04_bb_results-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "06jfJON-h096"
   },
   "source": [
    "# Analysis of Results of Anomaly Detection with Filtering by Binary Classifier\n",
    "Results of GEE VAE autoencoder combined with binary classifier filtering the traffic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lAX_3LHJ2X2h"
   },
   "source": [
    "## Environment Setting\n",
    "Import libraries/packages/modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H85hf1lp_bIE"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y7bH5s7BincW"
   },
   "source": [
    "## Load Results and Run Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bNEKwZ226fjW"
   },
   "outputs": [],
   "source": [
    "#setting the threshold\n",
    "\n",
    "#train background mean = 0.003405, train background std = 0.005413\n",
    "train_mean = 0.003405\n",
    "train_std = 0.005413\n",
    "\n",
    "coef = 1 #1/3, 1/2, 1, 2, 3\n",
    "threshold = train_mean + train_std * coef"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sXghXvW3MMjJ"
   },
   "source": [
    "### Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eOaDVNqfDnvK"
   },
   "outputs": [],
   "source": [
    "#function for enhancing classification results by results from anomaly detection\n",
    "def preprocess_results(ad_data, cf_data, threshold):\n",
    "   #take MSE from anomaly detection results and combine it with classification results\n",
    "   data = cf_data\n",
    "   data['mse'] = ad_data['mse']\n",
    "   #modify background-classified predictions based on MSE from AD crossing the threshold, label as \"anomaly\"\n",
    "   data.predictions[(data['predictions'] == '0') & (data['mse'] > threshold)] = 'anomaly'\n",
    "   return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4Di-wvkbKRKE"
   },
   "outputs": [],
   "source": [
    "#function for binarizing labels and predicitons (add to an input dataframe with labels and predictions columns blabels and bpredictions columns)\n",
    "def binarize_results(data):\n",
    "   #binarize labels\n",
    "   data['blabels'] = data['labels']\n",
    "   data.blabels[data['blabels'] == 'background'] = 0\n",
    "   data.blabels[data['blabels'] != 0] = 1\n",
    "   #binarize predictions\n",
    "   data['bpredictions'] = data['predictions']\n",
    "   data.bpredictions[data['bpredictions'] == '0'] = int(0)\n",
    "   data.bpredictions[data['bpredictions'] != 0] = int(1)\n",
    "   return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FTKHefadoVwf"
   },
   "outputs": [],
   "source": [
    "#function for printing AUC, classification report, and confusion matrix for given data (dataframe with blabels and bpredictions), title is a string to differentiate the printed results\n",
    "def print_result(data, title):\n",
    "   if(len(data.blabels.unique()) < 2):\n",
    "     print(title + ' subset contains a single label')\n",
    "     return\n",
    "   print('AUC (' + title + '):', metrics.roc_auc_score(data.blabels.to_list(), data.bpredictions.to_list()))\n",
    "   print(metrics.classification_report(data.blabels.to_list(), data.bpredictions.to_list(), digits=4))\n",
    "   print(metrics.confusion_matrix(data.blabels.to_list(), data.bpredictions.to_list()))\n",
    "   print('Anomalies by AD:', len(data[data['predictions'] == 'anomaly']))\n",
    "   print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IY_BLR4WRrQu"
   },
   "outputs": [],
   "source": [
    "#AUC ROC (based on MSE) from dataframe with binary labels (blabels) and MSE\n",
    "def plot_roc(data):\n",
    "   fig, ax = plt.subplots(figsize=(5, 5))\n",
    "   fpr, tpr, thresholds = metrics.roc_curve(data.blabels.to_list(), data.mse.to_list())\n",
    "   auc = metrics.auc(fpr, tpr)\n",
    "   ax.plot([0, 1], [0,1], 'k--')\n",
    "   ax.plot(fpr, tpr, label=f'MSE (AUC = {auc: .4f})')\n",
    "   ax.set_xlabel('False positive rate')\n",
    "   ax.set_ylabel('True positive rate')\n",
    "   ax.legend(loc='lower right')\n",
    "   fig.show()\n",
    "   return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HIuoFcwcTffN"
   },
   "outputs": [],
   "source": [
    "#confusion matrix from dataframe with binary labels and predictions (blabels and bpredictions)\n",
    "def plot_cm(data):\n",
    "   cmd = metrics.ConfusionMatrixDisplay(metrics.confusion_matrix(data.blabels.to_list(), data.bpredictions.to_list()), display_labels=['0','1'])\n",
    "   cmd.plot()\n",
    "   return cmd.figure_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QoB_7yC9Uw9a"
   },
   "outputs": [],
   "source": [
    "#KDE of MSE from dataframe with labels and MSE, MSE of 1.0 (filtered attacks) omitted from plotting\n",
    "def plot_kde(data):\n",
    "   normal_recon_error = data[(data['labels'] == 'background') & (data['mse'] != 1.0)]['mse'].tolist()\n",
    "   malicious_recon_error = data[(data['labels'] != 'background') & (data['mse'] != 1.0)]['mse'].tolist()\n",
    "   fig, ax = plt.subplots(figsize=(5, 5))\n",
    "   sns.kdeplot(normal_recon_error, ax=ax, label='Background MSE')\n",
    "   sns.kdeplot(malicious_recon_error, ax=ax, label='Anomaly-filtered MSE')\n",
    "   ax.legend(loc='lower right')\n",
    "   fig.show()\n",
    "   return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yVNIW9tTWfOR"
   },
   "outputs": [],
   "source": [
    "#histogram of MSE from dataframe with labels and MSE, MSE of 1.0 (filtered attacks) omitted from plotting\n",
    "def plot_hist(data):\n",
    "   normal_recon_error = data[(data['labels'] == 'background') & (data['mse'] != 1.0)]['mse'].tolist()\n",
    "   malicious_recon_error = data[(data['labels'] != 'background') & (data['mse'] != 1.0)]['mse'].tolist()\n",
    "   fig, ax = plt.subplots(figsize=(10, 5))\n",
    "   ax.hist([normal_recon_error, malicious_recon_error], bins=100, range=(0, 0.06), label=['Background', 'Anomaly-filtered'])\n",
    "   ax.set_ylim(0, 100)\n",
    "   ax.set_xlabel(\"MSE\")\n",
    "   ax.set_ylabel(\"Density\")\n",
    "   ax.legend(loc='upper right')\n",
    "   fig.show()\n",
    "   return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8dqK8jrbF1_r"
   },
   "source": [
    "### Binary classifier trained on all classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HsX8OzLjBX4m"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "ad_results = pd.read_feather('results_ad_test.feather.with_mse')\n",
    "cf_results = pd.read_feather('results_bb_test.feather')\n",
    "\n",
    "#combine classification and anomaly detection results\n",
    "results = preprocess_results(ad_results, cf_results, threshold)\n",
    "print(results.predictions.value_counts())\n",
    "\n",
    "#binarize labels and predictions\n",
    "binarize_results(results)\n",
    "print(results.blabels.value_counts())\n",
    "print(results.bpredictions.value_counts())\n",
    "\n",
    "print('Threshold = ', threshold)\n",
    "\n",
    "print_result(results, 'all')\n",
    "\n",
    "results_without_blacklist = results[results['labels'] != 'blacklist']\n",
    "print_result(results_without_blacklist, 'without blacklist')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eLPesWhNJeEU"
   },
   "source": [
    "### Binary classifier trained without blacklist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "X15T6qAx1iXc"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "ad_results = pd.read_feather('results_ad_test.feather.with_mse')\n",
    "cf_results = pd.read_feather('results_bb_without_blacklist_test.feather')\n",
    "\n",
    "#combine classification and anomaly detection results\n",
    "results = preprocess_results(ad_results, cf_results, threshold)\n",
    "print(results.predictions.value_counts())\n",
    "\n",
    "#binarize labels and predictions\n",
    "binarize_results(results)\n",
    "print(results.blabels.value_counts())\n",
    "print(results.bpredictions.value_counts())\n",
    "\n",
    "print('Threshold = ', threshold)\n",
    "\n",
    "print_result(results, 'all')\n",
    "\n",
    "results_without_blacklist = results[results['labels'] != 'blacklist']\n",
    "print_result(results_without_blacklist, 'without blacklist')\n",
    "\n",
    "print_result(results[(results['labels'] == 'background') | (results['labels'] == 'dos')], 'background + dos')\n",
    "print_result(results[(results['labels'] == 'background') | (results['labels'] == 'nerisbotnet')], 'background + nerisbotnet')\n",
    "print_result(results[(results['labels'] == 'background') | (results['labels'] == 'anomaly-spam')], 'background + anomaly-spam')\n",
    "print_result(results[(results['labels'] == 'background') | (results['labels'] == 'scan11')], 'background + scan11')\n",
    "print_result(results[(results['labels'] == 'background') | (results['labels'] == 'scan44')], 'background + scan44')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3qJ8DB1ZwXcz"
   },
   "outputs": [],
   "source": [
    "#modify for plotting (set MSE of classified attacks to 1.0)\n",
    "results_without_blacklist.mse[(results_without_blacklist['predictions'] == '1')] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LED_6pMqSb0K"
   },
   "outputs": [],
   "source": [
    "#plot ROC\n",
    "fig = plot_roc(results_without_blacklist)\n",
    "#fig.savefig('bcf-ad_auc-roc.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "djQ2d4kZ3KZY"
   },
   "outputs": [],
   "source": [
    "#plot confusion matrix\n",
    "fig = plot_cm(results_without_blacklist)\n",
    "#fig.savefig('bcf-ad_cm.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MavKLDnJKwoP"
   },
   "outputs": [],
   "source": [
    "#plot KDE\n",
    "fig = plot_kde(results_without_blacklist)\n",
    "fig.show()\n",
    "#fig.savefig('bcf-ad_kde.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "F11GXg4Uaxh4"
   },
   "outputs": [],
   "source": [
    "normal_recon_error = results_without_blacklist[(results_without_blacklist['labels'] == 'background')]['mse'].tolist()\n",
    "malicious_recon_error_filtered = results[(results['labels'] != 'blacklist') & (results['labels'] != 'background') & (results['predictions'] == '1')]['mse'].tolist()#\n",
    "fig, ax = plt.subplots(figsize=(5, 5))\n",
    "sns.kdeplot(normal_recon_error, ax=ax, label='Background MSE')\n",
    "sns.kdeplot(malicious_recon_error_filtered, ax=ax, label='Detected anomalies')\n",
    "ax.legend(loc='lower right')\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "X_wtUFhNY-53"
   },
   "outputs": [],
   "source": [
    "#plot histogram\n",
    "fig = plot_hist(results_without_blacklist)\n",
    "fig.show()\n",
    "#fig.savefig('bcf-ad_hist.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z5U_aCrJXTDU"
   },
   "source": [
    "### Binary classifier trained without blacklist and dos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cEnWjYTB9ixh"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "ad_results = pd.read_feather('results_ad.feather')\n",
    "cf_results = pd.read_feather('results_bb_without_blacklist_dos.feather')\n",
    "\n",
    "#combine classification and anomaly detection results\n",
    "results = preprocess_results(ad_results, cf_results, threshold)\n",
    "print(results.predictions.value_counts())\n",
    "\n",
    "#binarize labels and predictions\n",
    "binarize_results(results)\n",
    "print(results.blabels.value_counts())\n",
    "print(results.bpredictions.value_counts())\n",
    "\n",
    "print('Threshold = ', threshold)\n",
    "\n",
    "print_result(results, 'all')\n",
    "\n",
    "results_without_blacklist = results[results['labels'] != 'blacklist']\n",
    "print_result(results_without_blacklist, 'without blacklist')\n",
    "\n",
    "print_result(results[(results['labels'] == 'background') | (results['labels'] == 'dos')], 'background + dos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ca-SXCIwbZfm"
   },
   "outputs": [],
   "source": [
    "#modify for plotting (set MSE of classified attacks to 1.0)\n",
    "results_without_blacklist.mse[(results_without_blacklist['predictions'] == '1')] = 1.0\n",
    "\n",
    "fig = plot_roc(results_without_blacklist)\n",
    "fig = plot_cm(results_without_blacklist)\n",
    "fig = plot_kde(results_without_blacklist)\n",
    "fig = plot_hist(results_without_blacklist)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}