model_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ABD final project: Model training\n",
    "## Part 1: Data encoding\n",
    "### Python version\n",
    "3.5 onwards\n",
    "\n",
    "### Modules needed\n",
    "* pandas\n",
    "* numpy\n",
    "* xlrd\n",
    "* pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncoment to install modules inside the notebook\n",
    "# !pip install xlrd\n",
    "# !pip install pandas\n",
    "# !pip install pickle\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import random\n",
    "import re\n",
    "\n",
    "from io import StringIO\n",
    "from pandas import Series, DataFrame\n",
    "from collections import Counter\n",
    "\n",
    "import abd_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading and reordering data\n",
    "df_train = pd.read_excel('data/log_data_evil.xlsx', index_col=0)\n",
    "df_train.sort_index(ascending=True, inplace=True)\n",
    "\n",
    "# Droping columns that will not be used\n",
    "df_train.drop(['http_user_agent'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filtering and formatting the data with regex\n",
    "df_train.request_url.replace({r'^/$': '<BASE_URL>'}, regex=True, inplace=True)\n",
    "df_train.request_url.replace({r'\\%+[\\%0-9A-Za-z]*': ' <PERCENT_URL> '},regex=True, inplace=True)\n",
    "df_train.request_url.replace({r'\\w+\\d+\\w+': ' '},regex=True, inplace=True)\n",
    "df_train.request_url.replace({r'\\d{2,}': ' '},regex=True, inplace=True)\n",
    "df_train.request_url.replace({r'\\[\\d*\\]': ' '},regex=True, inplace=True)\n",
    "df_train.request_url.replace({'/': ' ', ':': ' ', '\\.': ' ', '\\?': ' ', '=': ' ', '\\|': ' ', '&': ' '},regex=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping extra spaces in the url strings\n",
    "list_aux = [' '.join(word.split()).lower() for word in df_train['request_url'].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a vocabulary for the neuronal network based on the url DataFrame column\n",
    "\n",
    "# Joinning all the url's in a String and separating all the individual words in an Array\n",
    "as_one = ' '.join(list_aux)\n",
    "words = as_one.split()\n",
    "\n",
    "# Dropping duplicate words\n",
    "counts = Counter(words)\n",
    "vocab = sorted(counts, key=counts.get)\n",
    "\n",
    "# Creating the vocabulary dictionary and predefining some reserved words\n",
    "word_index = {word: (index + 9) for (index, word) in enumerate(vocab)}\n",
    "word_index['<PAD>'] = 0\n",
    "word_index[\"<URL_START>\"] = 1\n",
    "word_index['<GET>'] = 2\n",
    "word_index['<POST>'] = 3\n",
    "word_index['<TEMPORARY_REDIRECT>'] = 4\n",
    "word_index['<BAD_REQUEST>'] = 5\n",
    "word_index['<NOT_FOUND>'] = 6\n",
    "word_index['<OK>'] = 7\n",
    "word_index['<MOVED_PERMANTLY>'] = 8\n",
    "\n",
    "# Creating the reverse vocabulary\n",
    "reverse_word_index = {value: key for (key, value) in word_index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving both vocabularys on disk\n",
    "with open('model/abd_variables.pkl', 'wb') as f:\n",
    "    pickle.dump([word_index, reverse_word_index], f)\n",
    "    \n",
    "# Saving word_index on disk\n",
    "with open('model/abd_w_index.pkl', 'wb') as f:\n",
    "    pickle.dump(word_index, f)\n",
    "    \n",
    "# Saving reverse_word_index on disk\n",
    "with open('model/abd_rw_index.pkl', 'wb') as f:\n",
    "    pickle.dump(reverse_word_index, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading functions for handling (encoding and decoding) the urls based on the vocabulary: ABDUtils class\n",
    "utils = abd_utils.ABDUtils(word_index, reverse_word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To lower case, then enconding and inserting a '<PAD>' encoded String to every url in the DataFrame\n",
    "df_train['request_url'] = df_train['request_url'].str.lower()\n",
    "df_train['request_url'] = df_train['request_url'].apply(utils.encode)\n",
    "df_train['request_url'] = df_train['request_url'].apply(abd_utils.insert_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding every request in the DataFrame\n",
    "df_train['request_method'].replace({'GET': 2, 'POST': 3}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From status code to String, then encoding every status code in the DataFrame\n",
    "df_train['status'].replace({307: '<TEMPORARY_REDIRECT>', 400: '<BAD_REQUEST>', 404: '<NOT_FOUND>', 200: '<OK>', 301: '<MOVED_PERMANTLY>'}, inplace=True)\n",
    "df_train['status'] = df_train['status'].apply(utils.encode_single)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separating and dropping the training labels from the DataFrame\n",
    "total_labels = df_train['is_evil'].values\n",
    "df_train.drop('is_evil', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the largest url have 11 words\n"
     ]
    }
   ],
   "source": [
    "# Getting the length of the the largest url in the DataFrame\n",
    "max_length = 0\n",
    "for i in df_train.values:\n",
    "    if len(i[2]) > max_length:\n",
    "        max_length = len(i[2])\n",
    "\n",
    "print('the largest url have', max_length, 'words')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ABD final project: Model training\n",
    "## Part 2: Model definition and training\n",
    "### Python version\n",
    "3.5 onwards\n",
    "\n",
    "### Modules needed\n",
    "* tensorflow\n",
    "* keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version: 1.14.0\n",
      "Available computing devices: ['/device:CPU:0', '/device:XLA_CPU:0']\n"
     ]
    }
   ],
   "source": [
    "# Model training: model definition and training\n",
    "\n",
    "# Importing modules\n",
    "\n",
    "# Uncoment to install modules inside the notebook\n",
    "# !pip install tensorflow\n",
    "\n",
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print('Tensorflow version:', tf.__version__)\n",
    "\n",
    "# Printing available computing devices \n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "def get_available_devices():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos]\n",
    "\n",
    "print('Available computing devices:', get_available_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalizing the url length\n",
    "aux_request_data = keras.preprocessing.sequence.pad_sequences(df_train['request_url'].values,\n",
    "                                                        value=word_index[\"<PAD>\"],\n",
    "                                                        padding='post',\n",
    "                                                        maxlen=max_length)\n",
    "df_train['request_url'] = Series(aux_request_data.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating 'max_length' new columns in the DataFrame\n",
    "for i in range(max_length):\n",
    "    df_train['request_url' + str(i)] = 0\n",
    "\n",
    "# Moving the url encoded data from its column to the new columns, one encoded String per column\n",
    "for i in df_train.itertuples():\n",
    "    for j in range(max_length):\n",
    "        df_train.at[i[0], 'request_url' + str(j)] = i[3][j]\n",
    "\n",
    "# Dropping the request url column in the DataFrame\n",
    "df_train.drop(['request_url'], axis=1, inplace=True)\n",
    "\n",
    "# Getting the total data Array from the values of the DataFrame\n",
    "total_data = df_train.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7716, 13) (7716,)\n",
      "727\n"
     ]
    }
   ],
   "source": [
    "# View the dimensions of the total data and labels variables\n",
    "print(total_data.shape, total_labels.shape)\n",
    "\n",
    "# Defining the vocabulary size\n",
    "vocab_size = len(word_index.keys())\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0718 19:50:44.891616 140694330087232 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0718 19:50:44.922627 140694330087232 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, None, 16)          11632     \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d (Gl (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 11,921\n",
      "Trainable params: 11,921\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Defining the neuronal network model\n",
    "model = keras.Sequential()\n",
    "model.add(keras.layers.Embedding(vocab_size, 16))\n",
    "model.add(keras.layers.GlobalAveragePooling1D())\n",
    "model.add(keras.layers.Dense(16, activation=tf.nn.relu))\n",
    "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n",
    "\n",
    "# Printing a summary of the model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0718 19:50:44.979377 140694330087232 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "# Compiling the model\n",
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Declaring 75% index of the total data\n",
    "half_length_data = int(len(total_data) * 0.75)\n",
    "\n",
    "# Separing the data: train data and labels (first 75% of total)\n",
    "train_data = total_data[:half_length_data]\n",
    "train_labels = total_labels[:half_length_data]\n",
    "\n",
    "# Separing the data: test data and labels (second 25% of total)\n",
    "test_data = total_data[half_length_data:]\n",
    "test_labels = total_labels[half_length_data:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data lengths: 5787 5787\n",
      "Test data lengths: 1929 1929\n"
     ]
    }
   ],
   "source": [
    "# Viewing the lengths of the train and test data\n",
    "print('Train data lengths:', len(train_data), len(train_labels))\n",
    "print('Test data lengths:', len(test_data), len(test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Declaring 40% index of the train data\n",
    "train_length = int(len(train_data) * 0.4)\n",
    "\n",
    "# Separing the train data: validation data (first 40% of the train data)\n",
    "x_val = train_data[:train_length]\n",
    "# Separing the train data: train data to fit the model (second 60% of the train data)\n",
    "partial_x_train = train_data[train_length:]\n",
    "\n",
    "# Separing the train labels: validation labels (first 40% of the train labels)\n",
    "y_val = train_labels[:train_length]\n",
    "# Separing the train labels: labels to fit the model (second 60% of the train labels)\n",
    "partial_y_train = train_labels[train_length:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit data and label lengths: 3473 3473\n",
      "Validate data and label lengths: 2314 2314\n"
     ]
    }
   ],
   "source": [
    "# Viewing the lengths of the separed train data\n",
    "print('Fit data and label lengths:', len(partial_x_train), len(partial_y_train))\n",
    "print('Validate data and label lengths:', len(x_val), len(y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Training the model\n",
    "history = model.fit(partial_x_train,\n",
    "                    partial_y_train,\n",
    "                    # Number of epochs to train the model\n",
    "                    # One epoch: one forward pass and one backward pass of all the training examples\n",
    "                    # One epoch: one pass over the entire dataset\n",
    "                    #epochs=250,\n",
    "                    epochs=len(partial_x_train)+len(x_val),\n",
    "                    # Number of samples per gradient update\n",
    "                    # 32 is the default size in the Keras framework\n",
    "                    # The higher the batch size, the more memory space needed\n",
    "                    # The smaller the batch size (until the length of the training data), the less accurate the estimate of the gradient will be\n",
    "                    #batch_size=32,\n",
    "                    batch_size=len(partial_x_train),\n",
    "                    validation_data=(x_val, y_val),\n",
    "                    # Printing the process: 0for silent, 1 for progress bar and 2 for one line per epoch\n",
    "                    verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1929/1929 [==============================] - 0s 16us/sample - loss: 7.3808e-04 - acc: 0.9995\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the model on the test data\n",
    "results = model.evaluate(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved to disk\n"
     ]
    }
   ],
   "source": [
    "# Saving the model to disk\n",
    "try:\n",
    "    model.save(\"model/abd_model.h5\")\n",
    "    print(\"model saved to disk\")\n",
    "except Exception:\n",
    "    print(\"something went wrong\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}