GroupAttribution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95f774ab-b9e3-48dc-88a1-90437524440b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import concurrent\n",
    "import concurrent.futures\n",
    "import ipaddress\n",
    "import pickle\n",
    "import re\n",
    "import statistics\n",
    "import warnings\n",
    "from collections import Counter\n",
    "\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import regex\n",
    "import torch\n",
    "import tqdm\n",
    "from featgenerator import group_features\n",
    "from featgenerator.config import Config\n",
    "from featgenerator.exif_feat import ExifFeatures\n",
    "from featgenerator.featurizer import Featurizer\n",
    "from featgenerator.floss_general_feat import FlossFeatures\n",
    "from featgenerator.floss_regex import FlossRegexFeatures\n",
    "from featgenerator.group_features import GroupAttributionFeatures\n",
    "from featgenerator.lief_features import LiefFeatures, get_features_from_function_lists\n",
    "from featgenerator.malcat import MalcatFeatures\n",
    "from featgenerator.util import ClusteringMetrics, DataProcessor, MinHashLSHForest, Util\n",
    "from itables import init_notebook_mode, show\n",
    "from keras import models\n",
    "from keras.layers import Dense, Input, Reshape\n",
    "from keras.models import Model\n",
    "from scipy.signal import normalize\n",
    "from sklearn.cluster import AffinityPropagation, AgglomerativeClustering, KMeans\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.metrics import (\n",
    "    adjusted_mutual_info_score,\n",
    "    adjusted_rand_score,\n",
    "    auc,\n",
    "    classification_report,\n",
    "    davies_bouldin_score,\n",
    "    roc_auc_score,\n",
    "    roc_curve,\n",
    "    silhouette_score,\n",
    "    v_measure_score,\n",
    ")\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import (\n",
    "    LabelEncoder,\n",
    "    QuantileTransformer,\n",
    "    label_binarize,\n",
    "    normalize,\n",
    ")\n",
    "from tqdm import tqdm\n",
    "from transformers import (\n",
    "    AutoModel,\n",
    "    AutoTokenizer,\n",
    "    BatchEncoding,\n",
    "    BertModel,\n",
    "    BertTokenizer,\n",
    "    PreTrainedModel,\n",
    "    PreTrainedTokenizer,\n",
    ")\n",
    "\n",
    "from featgenerator import util\n",
    "from importlib import reload\n",
    "util = reload(util)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72a3fa27-c725-444f-913e-24692cfdfe84",
   "metadata": {},
   "outputs": [],
   "source": [
    "from itables import init_notebook_mode\n",
    "\n",
    "init_notebook_mode(all_interactive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91ff26a2-5d92-4e55-af00-3b0d839fbc19",
   "metadata": {},
   "outputs": [],
   "source": [
    "from importlib import reload\n",
    "from featgenerator import util\n",
    "from typing import Optional, Tuple\n",
    "\n",
    "util = reload(util)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e05a36b5-ece4-4849-8ee6-63ad3bbdcd05",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load and prepare datasets\n",
    "exif_features, malcat_features, joined_df, adversary_dataset = group_features.load_and_prepare_datasets()\n",
    "# Process features and merge datasets\n",
    "final_features = group_features.process_and_merge_features(exif_features, malcat_features, joined_df, adversary_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78623f93-d63d-42f8-b5ce-8c41bc08a6ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate the string embeddings based on the floss features. We might not necessarily use this.\n",
    "floss_feat = FlossFeatures()\n",
    "string_embedding_processor = group_features.StringEmbeddingProcessor(joined_df=joined_df)\n",
    "string_embedding_df_features = string_embedding_processor.process()\n",
    "string_embedding_df_features.columns = string_embedding_df_features.columns.astype(str) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7426a1c6-8184-4236-b89e-8a63a911d92e",
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_adversary_experiment_final = adversary_dataset[['hash', 'Normalized_Tag']].merge(joined_df, on=\"hash\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c516f2c-5f0e-4868-9293-3d35838a2538",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = joined_df.merge(adversary_dataset, on = \"hash\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f86bd6db-f693-4bdc-bc09-fac9c4b1d247",
   "metadata": {},
   "outputs": [],
   "source": [
    "feat = final_features.drop(columns=['Normalized_Tag'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7876c16-743a-4275-b427-d29cd1f2c73e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "n_clusters= list(np.arange(5, 120, 5))\n",
    "modelling = util.Modelling()\n",
    "# To combine the string embedding with our features from the Group Attribution pipeline\n",
    "all_params, best_param, best_truth_matrix = modelling.find_best_agglo(pd.concat([feat, string_embedding_df_features], axis=1), n_clusters,all_features[['hash', 'Normalized_Tag']], 'Normalized_Tag')\n",
    "# Results without the string embeddings\n",
    "# all_params, best_param, best_truth_matrix = modelling.find_best_agglo(feat, n_clusters,all_features[['hash', 'Normalized_Tag']], 'Normalized_Tag')\n",
    "# Results with just the string embeddings\n",
    "# all_params, best_param, best_truth_matrix = modelling.find_best_agglo(string_embedding_df_features, n_clusters,all_features[['hash', 'Normalized_Tag']], 'Normalized_Tag')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caed99fc-416e-429c-9435-fae4631791cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "show(pd.DataFrame(all_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c299653c-b642-4c60-9b6a-deaae3fcada7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}