CampaignExecutableDomain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28f465d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import concurrent\n",
    "import concurrent.futures\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "import statistics\n",
    "import warnings\n",
    "\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import regex\n",
    "import tqdm\n",
    "from featgenerator.config import Config\n",
    "from featgenerator.exif_feat import ExifFeatures\n",
    "from featgenerator.featurizer import Featurizer\n",
    "from featgenerator.floss_general_feat import FlossFeatures\n",
    "from featgenerator.floss_regex import FlossRegexFeatures\n",
    "from featgenerator.lief_features import LiefFeatures, get_features_from_function_lists\n",
    "from featgenerator.malcat import MalcatFeatures\n",
    "from featgenerator import feature_transformation\n",
    "from featgenerator.util import ClusteringMetrics, DataProcessor, MinHashLSHForest\n",
    "from itables import init_notebook_mode, show\n",
    "from keras import models\n",
    "from keras.layers import Dense, Input, Reshape\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from sklearn.cluster import AffinityPropagation, AgglomerativeClustering, KMeans\n",
    "from sklearn.feature_extraction.text import (\n",
    "    CountVectorizer,\n",
    "    TfidfTransformer,\n",
    "    TfidfVectorizer,\n",
    ")\n",
    "from sklearn.metrics import (\n",
    "    adjusted_mutual_info_score,\n",
    "    adjusted_rand_score,\n",
    "    auc,\n",
    "    classification_report,\n",
    "    davies_bouldin_score,\n",
    "    roc_auc_score,\n",
    "    roc_curve,\n",
    "    silhouette_score,\n",
    "    v_measure_score,\n",
    ")\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import (\n",
    "    LabelEncoder,\n",
    "    QuantileTransformer,\n",
    "    label_binarize,\n",
    "    normalize,\n",
    ")\n",
    "from featgenerator.util import Util\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from importlib import reload\n",
    "from featgenerator import util\n",
    "from featgenerator import util\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6510a67f-9745-4c2f-9257-b70b0bbdbb32",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:1024'\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from itables import init_notebook_mode\n",
    "\n",
    "init_notebook_mode(all_interactive=True)\n",
    "\n",
    "util = reload(util)\n",
    "feature_transformation = reload(feature_transformation)\n",
    "\n",
    "# Reload the util module if you make changes\n",
    "# util = reload(util)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8573ed77-e1de-4805-97bb-b02bee9cf646",
   "metadata": {},
   "source": [
    "## Before loading config make sure you have the right root_dir in the config file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fed45d36",
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = Config()\n",
    "\n",
    "adversary_dataset = pd.read_csv(conf.get_adversary_mapping())\n",
    "adversary_dataset['hash'] = adversary_dataset['sha256'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f619c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# To get feature specific information you can get the keys and get the downstream keys\n",
    "# E.g. \n",
    "# >>> joined_data_dict.keys()\n",
    "# dict_keys(['floss_features', 'exif_features', 'malcat_features', 'lief_features', 'flossregex_features', 'exported_functions_features', 'configuration_version'])\n",
    "# To get the exported function features\n",
    "# >>> exdf = joined_data_dict['exported_functions_features']\n",
    "\n",
    "joined_df, joined_data_dict = feature_transformation.get_combined_features(\n",
    "    floss_features=False,\n",
    "    exif_features=True,\n",
    "    malcat_features=True,\n",
    "    lief_features=True,\n",
    "    flossregex_features=True,\n",
    "    exported_functions_features=True,\n",
    "    configuration_version=True\n",
    ")\n",
    "lcf = joined_data_dict['lief_features']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0958ef7b-bbdd-4f6b-a332-8d512e3839fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_df = Util().process_raw_strings_dataset(conf.get_root_dir(), conf.get_floss_file(), joined_df['hash'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2c1b279-6377-497b-9d3e-8f5fc8dcfa48",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_df['hash'] = joined_df['hash'].astype(str).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b844049",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_inner = joined_df.merge(embedding_df,how=\"inner\", on=[\"hash\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10042071-19bd-4e31-84d2-562c618cec43",
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder, X_encoded, combined_features, all_features  = feature_transformation.prepare_and_encode_features(joined_df, embedding_df, adversary_dataset, target_column=\"Campaign_Tag\", include_embedding=False, num_epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e1850bd-ebd4-4b03-89c5-a6801819963d",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_clusters = list(np.arange(5, 60, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1914ac4-0cd9-4188-acd7-52bdc7710c0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "modelling = util.Modelling()\n",
    "all_params, best_param, best_truth_matrix = modelling.find_best_agglo(combined_features, n_clusters,all_features[['hash', 'Campaign_Tag']], 'Campaign_Tag')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b567b10d-0714-444a-9103-9a0ee7883a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "show(pd.DataFrame(all_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3469fda1-192a-4c6f-9eda-5269fc36fe7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "agglomerative = AgglomerativeClustering(n_clusters=55, metric=\"euclidean\", linkage=\"ward\")\n",
    "y_pred = agglomerative.fit_predict(combined_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f588ec9d-5645-4f51-9626-385f298b2f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "merg_res = all_features.copy()\n",
    "merg_res['labels'] = y_pred\n",
    "\n",
    "merged_adversary_experiment_final = all_features.copy()\n",
    "merged_adversary_experiment_final['labels'] = y_pred\n",
    "clustering_metrics = util.ClusteringMetrics()\n",
    "truth_matrix, precision, recall = clustering_metrics.metrics_using_labels(merged_adversary_experiment_final, evaluation_column = 'Campaign_Tag')\n",
    "truth_matrix.fillna('0', inplace=True)\n",
    "evaluation_column = 'Campaign_Tag'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0475aef2-3d6b-4caa-a6b9-6bb09c40b5d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "truth_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b42e589-9a05-4d51-bddd-be8cf9bb15e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "truth_matrix[truth_matrix['Campaign_Tag'] == 'UNC4990Jan2024']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}