text_norm_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "from nltk import FreqDist\n",
    "import time\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, Embedding, Bidirectional\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.layers.recurrent import LSTM\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_vocab_size = 250\n",
    "target_vocab_size = 346    # 1000 in full dataset\n",
    "# target_vocab_size = 1000\n",
    "num_samples = 500000\n",
    "context_size = 3\n",
    "padding_entity = [0]\n",
    "self_sil_retention_percent = 0.5\n",
    "X_seq_len = 60\n",
    "y_seq_len = 20\n",
    "hidden = 256\n",
    "layers = 2\n",
    "epochs = 5    # used just 1 for full dataset\n",
    "batch_size = 128\n",
    "val_split = 0.1\n",
    "learning_rate = 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM encoder/decoder for text normalization\n",
    "\n",
    "Given a large corpus of written text aligned to its normalized spoken form, I will train an RNN to learn the correct normalization function. My model is a encoder-decoder with layers of bidirectional LSTMs.\n",
    "\n",
    "Here is a quick summary of my model approach and architecture:\n",
    "\n",
    "+ Character level input sequence, and word level output sequence\n",
    "+ Input character vocabulary of 250 distinct characters. Output word vocabulary of 346 (the number of unique) distinct words.\n",
    "+ Add a context window of 3 words to the left and right with a distinctive tag to separately identity the key token. I do this to manage the input sequence length reasonably.\n",
    "+ Input sequence zero padding to a maximum of length 60. Output sequence padding to a maximum of length 20. I do this to create fixed-length sequences.\n",
    "+ Model architecture with two components: an encoder and a decoder.\n",
    "  - 256 hidden units in each layer of the encoder and decoder\n",
    "  - Three bidirectional LSTM layers in the encoder\n",
    "  - Two LSTM layers in the decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Create the encoder/decoder model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding layer created\n",
      "Encoder layer created\n",
      "Decoder layer created\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 60, 256)           64512     \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 60, 512)           1050624   \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 60, 512)           1574912   \n",
      "_________________________________________________________________\n",
      "bidirectional_3 (Bidirection (None, 512)               1574912   \n",
      "_________________________________________________________________\n",
      "repeat_vector_1 (RepeatVecto (None, 20, 512)           0         \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 20, 256)           787456    \n",
      "_________________________________________________________________\n",
      "lstm_5 (LSTM)                (None, 20, 256)           525312    \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 20, 347)           89179     \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 20, 347)           0         \n",
      "=================================================================\n",
      "Total params: 5,666,907\n",
      "Trainable params: 5,666,907\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# let's make the model\n",
    "model = Sequential()\n",
    "\n",
    "# creating encoder network\n",
    "model.add(Embedding(input_vocab_size+2, hidden, input_length=X_seq_len, mask_zero=True))\n",
    "print('Embedding layer created')\n",
    "model.add(Bidirectional(LSTM(hidden, return_sequences = True), merge_mode = 'concat'))\n",
    "model.add(Bidirectional(LSTM(hidden, return_sequences = True), merge_mode = 'concat'))\n",
    "model.add(Bidirectional(LSTM(hidden), merge_mode = 'concat'))\n",
    "model.add(RepeatVector(y_seq_len))\n",
    "print('Encoder layer created')\n",
    "\n",
    "# creating decoder network\n",
    "for _ in range(layers):\n",
    "    model.add(LSTM(hidden, return_sequences=True))\n",
    "model.add(TimeDistributed(Dense(target_vocab_size+1)))\n",
    "model.add(Activation('softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "print('Decoder layer created')\n",
    "\n",
    "# checking the model summary\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Load the training data and do a quick exploratory analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data loaded in 9.865117311477661 s.\n",
      "(9918441, 5)\n",
      "(500000, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>class</th>\n",
       "      <th>before</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>Brillantaisia</td>\n",
       "      <td>Brillantaisia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>is</td>\n",
       "      <td>is</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>genus</td>\n",
       "      <td>genus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>of</td>\n",
       "      <td>of</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentence_id  token_id  class         before          after\n",
       "0            0         0  PLAIN  Brillantaisia  Brillantaisia\n",
       "1            0         1  PLAIN             is             is\n",
       "2            0         2  PLAIN              a              a\n",
       "3            0         3  PLAIN          genus          genus\n",
       "4            0         4  PLAIN             of             of"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# load training data\n",
    "X_train_data = pd.read_csv(\"en_train.csv\")\n",
    "X_train_data['before'] = X_train_data['before'].apply(str)\n",
    "X_train_data['after'] = X_train_data['after'].apply(str)\n",
    "\n",
    "print('Training data loaded in {0} s.'.format(time.time()-start))\n",
    "\n",
    "print(X_train_data.shape)\n",
    "X_train_data = X_train_data.iloc[:num_samples]\n",
    "print(X_train_data.shape)\n",
    "X_train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1358eda58>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8kAAAF3CAYAAACWkwm1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xu4pXVd9/H3l+EgVshpQmTAQZxSREWdgEx7EAoGywYLFJ6UkUgsoad81AB7Ck+UpkjhgcIYAVORMGSswYk4KJYcBhmBAZURUZg4jAxihKLg9/nj/u7hnjVrrb32Ye09e/t+Xde69lq/+/T7rfv4WfdhR2YiSZIkSZJgi+mugCRJkiRJmwtDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJElly+muwOZi5513zvnz5093NSRJkiRJQ3DDDTd8NzPnjtafIbnMnz+flStXTnc1JEmSJElDEBHfHqQ/L7eWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKlsOd0V2NytO+sfp7sKYzL3D18z3VWQJEmSpBnLM8mSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEllaCE5Ip4UEddFxFcjYnVEvKPKz42Ib0XEqnrtW+UREWdGxJqIuCkiXtga15KIuL1eS1rlL4qIm2uYMyMiqnzHiLis+r8sInYYVjslSZIkSbPHMM8kPwoclJnPB/YFFkXEAdXtrZm5b71WVdlhwIJ6HQ+cBU3gBU4F9gf2A05thd6zgNe3hltU5ScDl2fmAuDy+ixJkiRJUl9DC8nZeLg+blWv7DPIYuD8Gu4aYPuI2BU4FLgsM9dn5oPAZTSBe1dgu8y8JjMTOB84vDWu8+r9ea1ySZIkSZJ6Guo9yRExJyJWAffTBN1rq9NpdUn1GRGxTZXtBtzVGvzuKutXfneXcoBdMvOeen8vsMtktUmSJEmSNHsNNSRn5uOZuS8wD9gvIvYBTgGeBfwSsCNw0pDrkPQ4gx0Rx0fEyohYuW7dumFWQ5IkSZI0A0zJ060z83vAlcCizLynLql+FPgYzX3GAGuB3VuDzauyfuXzupQD3FeXY1N/7+9Rr7Mzc2FmLpw7d+5EmihJkiRJmgWG+XTruRGxfb3fFvh14Gut8Bo09wrfUoMsA46pp1wfADxUl0yvAA6JiB3qgV2HACuq2/cj4oAa1zHAJa1xjTwFe0mrXJIkSZKknrYc4rh3Bc6LiDk0YfzCzPyXiLgiIuYCAawC/qD6Xw68HFgDPAIcC5CZ6yPiXcD11d87M3N9vX8jcC6wLXBpvQDeA1wYEccB3wZeNbRWSpIkSZJmjaGF5My8CXhBl/KDevSfwAk9ui0FlnYpXwns06X8AeDgMVZZkiRJkvRTbkruSZYkSZIkaSYwJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUhlaSI6IJ0XEdRHx1YhYHRHvqPI9I+LaiFgTEZ+OiK2rfJv6vKa6z2+N65Qq/3pEHNoqX1RlayLi5FZ512lIkiRJktTPMM8kPwoclJnPB/YFFkXEAcB7gTMy85nAg8Bx1f9xwINVfkb1R0TsDRwFPAdYBHwkIuZExBzgw8BhwN7A0dUvfaYhSZIkSVJPQwvJ2Xi4Pm5VrwQOAi6q8vOAw+v94vpMdT84IqLKL8jMRzPzW8AaYL96rcnMOzLzR8AFwOIaptc0JEmSJEnqaaj3JNcZ31XA/cBlwDeB72XmY9XL3cBu9X434C6A6v4QsFO7vGOYXuU79ZmGJEmSJEk9DTUkZ+bjmbkvMI/mzO+zhjm9sYqI4yNiZUSsXLdu3XRXR5IkSZI0zabk6daZ+T3gSuCXge0jYsvqNA9YW+/XArsDVPenAA+0yzuG6VX+QJ9pdNbr7MxcmJkL586dO6E2SpIkSZJmvmE+3XpuRGxf77cFfh24jSYsH1G9LQEuqffL6jPV/YrMzCo/qp5+vSewALgOuB5YUE+y3prm4V7Laphe05AkSZIkqactR+9l3HYFzqunUG8BXJiZ/xIRtwIXRMS7gRuBc6r/c4CPR8QaYD1N6CUzV0fEhcCtwGPACZn5OEBEnAisAOYASzNzdY3rpB7TkCRJkiSpp6GF5My8CXhBl/I7aO5P7iz/IXBkj3GdBpzWpXw5sHzQaUiSJEmS1M+U3JMsSZIkSdJMYEiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqQytJAcEbtHxJURcWtErI6IP67yt0fE2ohYVa+Xt4Y5JSLWRMTXI+LQVvmiKlsTESe3yveMiGur/NMRsXWVb1Of11T3+cNqpyRJkiRp9hjmmeTHgDdn5t7AAcAJEbF3dTsjM/et13KA6nYU8BxgEfCRiJgTEXOADwOHAXsDR7fG894a1zOBB4Hjqvw44MEqP6P6kyRJkiSpr6GF5My8JzO/Uu//G7gN2K3PIIuBCzLz0cz8FrAG2K9eazLzjsz8EXABsDgiAjgIuKiGPw84vDWu8+r9RcDB1b8kSZIkST1NyT3JdbnzC4Brq+jEiLgpIpZGxA5VthtwV2uwu6usV/lOwPcy87GO8o3GVd0fqv4lSZIkSepp6CE5In4W+AzwJ5n5feAsYC9gX+Ae4PRh16FP3Y6PiJURsXLdunXTVQ1JkiRJ0mZiqCE5IraiCcifyMx/BsjM+zLz8cz8CfBRmsupAdYCu7cGn1dlvcofALaPiC07yjcaV3V/SvW/kcw8OzMXZubCuXPnTrS5kiRJkqQZbphPtw7gHOC2zPxAq3zXVm+vBG6p98uAo+rJ1HsCC4DrgOuBBfUk661pHu61LDMTuBI4ooZfAlzSGteSen8EcEX1L0mSJElST1uO3su4/QrwWuDmiFhVZW+jeTr1vkACdwJvAMjM1RFxIXArzZOxT8jMxwEi4kRgBTAHWJqZq2t8JwEXRMS7gRtpQjn19+MRsQZYTxOsJUmSJEnqa2ghOTO/BHR7ovTyPsOcBpzWpXx5t+Ey8w6euFy7Xf5D4Mix1FeSJEmSpCl5urUkSZIkSTOBIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqA4XkiLh8kDJJkiRJkmayLft1jIgnAU8Gdo6IHYCoTtsBuw25bpIkSZIkTanRziS/AbgBeFb9HXldAnyo34ARsXtEXBkRt0bE6oj44yrfMSIui4jb6+8OVR4RcWZErImImyLiha1xLan+b4+IJa3yF0XEzTXMmRER/aYhSZIkSVI/fUNyZv5tZu4JvCUzn5GZe9br+ZnZNyQDjwFvzsy9gQOAEyJib+Bk4PLMXABcXp8BDgMW1Ot44CxoAi9wKrA/sB9waiv0ngW8vjXcoirvNQ1JkiRJknrqe7n1iMz8YES8GJjfHiYzz+8zzD3APfX+vyPiNppLtBcDB1Zv5wFXASdV+fmZmcA1EbF9ROxa/V6WmesBIuIyYFFEXAVsl5nXVPn5wOHApX2mIUmSJElSTwOF5Ij4OLAXsAp4vIoT6BmSO4afD7wAuBbYpQI0wL3ALvV+N+Cu1mB3V1m/8ru7lNNnGpIkSZIk9TRQSAYWAnvXWd4xiYifBT4D/Elmfr9uGwYgMzMixjzOseg3jYg4nubSbvbYY49hVkOSJEmSNAMM+n+SbwGeOtaRR8RWNAH5E5n5z1V8X11GTf29v8rXAru3Bp9XZf3K53Up7zeNjWTm2Zm5MDMXzp07d6zNkyRJkiTNMoOG5J2BWyNiRUQsG3n1G6CeNH0OcFtmfqDVaRkw8oTqJTRPyh4pP6aecn0A8FBdMr0COCQidqgHdh0CrKhu34+IA2pax3SMq9s0JEmSJEnqadDLrd8+jnH/CvBa4OaIWFVlbwPeA1wYEccB3wZeVd2WAy8H1gCPAMcCZOb6iHgXcH31986Rh3gBbwTOBbaleWDXpVXeaxqSJEmSJPU06NOtvzDWEWfml4Do0fngLv0ncEKPcS0FlnYpXwns06X8gW7TkCRJkiSpn0Gfbv3fNE+zBtga2Ar4n8zcblgVkyRJkiRpqg16JvnnRt7X/b+LgQOGVSlJkiRJkqbDoA/u2iAbnwUOHUJ9JEmSJEmaNoNebv3brY9b0Pzf5B8OpUaSJEmSJE2TQZ9u/YrW+8eAO2kuuZYkSZIkadYY9J7kY4ddEUmSJEmSpttA9yRHxLyIuDgi7q/XZyJi3rArJ0mSJEnSVBr0wV0fA5YBT6vX56pMkiRJkqRZY9CQPDczP5aZj9XrXGDuEOslSZIkSdKUGzQkPxARr4mIOfV6DfDAMCsmSZIkSdJUGzQk/x7wKuBe4B7gCOB1Q6qTJEmSJEnTYtB/AfVOYElmPggQETsC76cJz5IkSZIkzQqDnkl+3khABsjM9cALhlMlSZIkSZKmx6AheYuI2GHkQ51JHvQstCRJkiRJM8KgQfd04MsR8U/1+UjgtOFUSZIkSZKk6TFQSM7M8yNiJXBQFf12Zt46vGpJkiRJkjT1Br5kukKxwViSJEmSNGsNek+yJEmSJEmzniFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqQwvJEbE0Iu6PiFtaZW+PiLURsapeL291OyUi1kTE1yPi0Fb5oipbExEnt8r3jIhrq/zTEbF1lW9Tn9dU9/nDaqMkSZIkaXYZ5pnkc4FFXcrPyMx967UcICL2Bo4CnlPDfCQi5kTEHODDwGHA3sDR1S/Ae2tczwQeBI6r8uOAB6v8jOpPkiRJkqRRDS0kZ+YXgfUD9r4YuCAzH83MbwFrgP3qtSYz78jMHwEXAIsjIoCDgItq+POAw1vjOq/eXwQcXP1LkiRJktTXdNyTfGJE3FSXY+9QZbsBd7X6ubvKepXvBHwvMx/rKN9oXNX9oepfkiRJkqS+pjoknwXsBewL3AOcPsXT30hEHB8RKyNi5bp166azKpIkSZKkzcCUhuTMvC8zH8/MnwAfpbmcGmAtsHur13lV1qv8AWD7iNiyo3yjcVX3p1T/3epzdmYuzMyFc+fOnWjzJEmSJEkz3JSG5IjYtfXxlcDIk6+XAUfVk6n3BBYA1wHXAwvqSdZb0zzca1lmJnAlcEQNvwS4pDWuJfX+COCK6l+SJEmSpL62HL2X8YmITwEHAjtHxN3AqcCBEbEvkMCdwBsAMnN1RFwI3Ao8BpyQmY/XeE4EVgBzgKWZubomcRJwQUS8G7gROKfKzwE+HhFraB4cdtSw2ihJkiRJml2GFpIz8+guxed0KRvp/zTgtC7ly4HlXcrv4InLtdvlPwSOHFNlJUmSJEliep5uLUmSJEnSZsmQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJZWghOSKWRsT9EXFLq2zHiLgsIm6vvztUeUTEmRGxJiJuiogXtoZZUv3fHhFLWuUvioiba5gzIyL6TUOSJEmSpNEM80zyucCijrKTgcszcwFweX0GOAxYUK/jgbOgCbzAqcD+wH7Aqa3Qexbw+tZwi0aZhiRJkiRJfQ0tJGfmF4H1HcWLgfPq/XnA4a3y87NxDbB9ROwKHApclpnrM/NB4DJgUXXbLjOvycwEzu8YV7dpSJIkSZLU11Tfk7xLZt5T7+8Fdqn3uwF3tfq7u8r6ld/dpbzfNCRJkiRJ6mvaHtxVZ4BzOqcREcdHxMqIWLlu3bphVkWSJEmSNANMdUi+ry6Vpv7eX+Vrgd1b/c2rsn7l87qU95vGJjLz7MxcmJkL586dO+5GSZIkSZJmh6kOycuAkSdULwEuaZUfU0+5PgB4qC6ZXgEcEhE71AO7DgFWVLfvR8QB9VTrYzrG1W0akiRJkiT1teWwRhwRnwIOBHaOiLtpnlL9HuDCiDgO+Dbwqup9OfByYA3wCHAsQGauj4h3AddXf+/MzJGHgb2R5gna2wKX1os+01CHez5y0nRXYcx2feN7p7sKkiRJkmaxoYXkzDy6R6eDu/SbwAk9xrMUWNqlfCWwT5fyB7pNQ5IkSZKk0Uzbg7skSZIkSdrcGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqRiSJYkSZIkqRiSJUmSJEkqhmRJkiRJkoohWZIkSZKkYkiWJEmSJKkYkiVJkiRJKtMSkiPizoi4OSJWRcTKKtsxIi6LiNvr7w5VHhFxZkSsiYibIuKFrfEsqf5vj4glrfIX1fjX1LAx9a2UJEmSJM0003km+WWZuW9mLqzPJwOXZ+YC4PL6DHAYsKBexwNnQROqgVOB/YH9gFNHgnX18/rWcIuG3xxJkiRJ0ky3OV1uvRg4r96fBxzeKj8/G9cA20fErsChwGWZuT4zHwQuAxZVt+0y85rMTOD81rgkSZIkSeppukJyAv8WETdExPFVtktm3lPv7wV2qfe7AXe1hr27yvqV392lXJIkSZKkvracpum+JDPXRsTPA5dFxNfaHTMzIyKHXYkK6McD7LHHHsOenCRJkiRpMzctZ5Izc239vR+4mOae4vvqUmnq7/3V+1pg99bg86qsX/m8LuXd6nF2Zi7MzIVz586daLMkSZIkSTPclIfkiPiZiPi5kffAIcAtwDJg5AnVS4BL6v0y4Jh6yvUBwEN1WfYK4JCI2KEe2HUIsKK6fT8iDqinWh/TGpckSZIkST1Nx+XWuwAX139l2hL4ZGZ+PiKuBy6MiOOAbwOvqv6XAy8H1gCPAMcCZOb6iHgXcH31987MXF/v3wicC2wLXFovSZIkSZL6mvKQnJl3AM/vUv4AcHCX8gRO6DGupcDSLuUrgX0mXFlJkiRJ0k+VzelfQEmSJEmSNK0MyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElSMSRLkiRJklQMyZIkSZIkFUOyJEmSJEnFkCxJkiRJUjEkS5IkSZJUDMmSJEmSJBVDsiRJkiRJxZAsSZIkSVIxJEuSJEmSVAzJkiRJkiQVQ7IkSZIkScWQLEmSJElS2XK6KyANy/V//4rprsKY/dIbPjfdVZAkSZJ+qnkmWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpOLTraUZ6pKlh013FcZs8e9dOt1VkCRJkvoyJEvaLP39xw+d7iqMyRteu2K6qyBJkqRJ4OXWkiRJkiSVWXsmOSIWAX8LzAH+ITPfM81VkqQNTrpo0XRXYUzee8Tnp7sKkiRJU2JWnkmOiDnAh4HDgL2BoyNi7+mtlSRJkiRpczcrQzKwH7AmM+/IzB8BFwCLp7lOkiRJkqTN3Gy93Ho34K7W57uB/aepLpL0U+Xln33zdFdhzJYffvrA/f7GP39wiDUZjn/97T+a7ipsNhZfNLOesn/JETPvPxlI0kwXmTnddZh0EXEEsCgzf78+vxbYPzNP7OjveOD4+viLwNensJo7A9+dwulNtdncvtncNrB9M53tm7lmc9vA9s10tm/mms1tA9s30011+56emXNH62m2nkleC+ze+jyvyjaSmWcDZ09VpdoiYmVmLpyOaU+F2dy+2dw2sH0zne2buWZz28D2zXS2b+aazW0D2zfTba7tm633JF8PLIiIPSNia+AoYNk010mSJEmStJmblWeSM/OxiDgRWEHzL6CWZubqaa6WJEmSJGkzNytDMkBmLgeWT3c9+piWy7yn0Gxu32xuG9i+mc72zVyzuW1g+2Y62zdzzea2ge2b6TbL9s3KB3dJkiRJkjQes/WeZEmSJEmSxi4zfU3gBTwOrAJuAf4JeHKVP9xnmFXABR1l5wJH1PurgJWtbguBq6a7XcB84JaO/t4OvKXVhrXANvV5Z+DOVr+/QHMJ/O3AV4ALgVfXdFYBD9P8G65VwPnT0NbVwFeBNwNbdPTzN9W2Lerzsa16/wi4ud6/B3gdsK7VfRWw9xDrv8myVvNlbUcden3X9/SaB8CBwEMd4/m1LsvI54Dtq3wL4Mwqv5nmQXp7jqE9TwUuAL4J3FDLzC9Utz8Bfgg8pdV/u45fA97f6jYyL26s5W4F8OLxrnedy0FrGh8aoF1XAod2lP0JcCnwg47v+Jjqfmd9hzcBX6D5twWdy+1XadanF3cZ94bvCjh0lPn8L6325Mh8rrLDq+yIyVivRlmuus5/WtufGj6B329Nb98qe0urbMua/+/pqNtVwMJxrm8J/GOXabS/v57rf+d8qbInA5+oeX0L8CXgZxlsm/ut1nJwcEcbv96qw0UT3M70bXdrObkJuK3acnjHutZ1/1Dt3GQdqO/kD1vj2L/Gv9VE2lLjmgdcQrNd+Cbwt8DWzJDtyWS8gJ1a3/e9bLzPeKQ1bxJ4d2u4nYEfj9ST7vub7cdZp8c7xnNyr3WWsW9Hnt/qbz1PrDv/3rEM3kqzXdyqNa2XANfVMvE14PiOdfIR4OdbZQ/3eN/tOGiXCXxPvbav7fVyUavuq4BPA3u0l1ng4uq2puM7ffFY6zbgfJ1Pn3WtNdxngWu6lB/DE8cYNwJvAT7cmn/t7ckRbLxubk2z7q2p+XAJMK817gROb31+C/D2AdaZZHKW3YFyBR3bCpp/aTuyfF4HvKTV7Sp6bI/61aXL9z5yLPCs1vbhBzUPbqvpvq6jjqNtM8e0D6P5l71XVdltwNlV3nU/OqHldjIW/p/mFxtv/D4B/N9uC3Orn2fXDFwL/EzHgtLeuX4HOKxzYZ7OdjHYAdt3qIMaNj4IelKtIK9oDXsgsE/r81WM88B1Etv68zQ7zHe0yrYAvg1cA7ysy/B3Aju3Pr+OKTrI6bWstedLj2G6fted5XTsbPt8b+cBf1bvjwYu4okd9jxghwHbEsCXgT9olT0feGm9vxa4Gji2Wx2BbWl2EL/SbV4AL6PZsT27tcwOtN71Wg4Gnd80O7CPdZRdA/xq53rVbdkC3gF8tMf3fyjwhY5hN/muBpnP1Z6bgH9odf80dbAxnuWSjvWq13LVb/6zaUi+Gfi3Vn/vrTq2Q/JhwH/QHCjHaMv/oO2q6Wzbmsaqju+v5/LQbb4ApwAfaH3+RWAbBtvmjiy/LwNun4w2jrPdz6c56NyzPu9Zn5/Xqmuv/cMm7azyXYA7gLk069/1tA78JtCWoDmYO7Y+zwHOAd7HDNmeTPaLjn0Gtf7WvLkDuLHV7Q9r3n+o27ATXc56lG+yPDOO7Ujr84Z51bkM1vJwBfC79fmpNS9f2Fp2bwB+o9X+7wDv7daO1nc56nHQeL4n+mxfgX1qms9u9f9bwK/2+B66fqfDmK/0WdeqbHvgLpog9IxW+WE0PzA8rT5vA7y+27zsNr+B99Os73Pq87E024ORW1B/SBPcRva9G0LyIOvMRJbdLvO2Z66gta0AfrOWyZE6v7CWyae26tB1ezSWeU5zLHB1a1mfdVbpAAAN7UlEQVTb6LsGnkGzbTi2s471ud82c6B9GE3QXtz6/Nz623U/OpHl1sutJ9fVwDNH6edo4OPAvwGL+/T3PuDPJqleEzVIu0b8DfCmiOh8KNz/Br6cmZ8bKcjMqzLzlkmq46TIzPtpwsyJERFVfCDNr7Vn0cw/berLwG71flfgnsz8CUBm3p2ZDw44npcBP87MvxspyMyvZubVEbEXzdm1/0eP+ZCZI78e79aj+5U0D4g4vsf0+613BzKx5eAi4Dfq39IREfOBp9EcBAyi/R132g7Y8B0P8l2N4mpgv4jYKiJ+lmb9XzWO8QA916tues7/Lv1+G3hSROxS41xEc1a+7WiaM4TfAX55vPXvYjnwG61pfGqQgfrMl11pfjgFIDO/npmPjrFO/ZaPydKv3W8B/jIzvwVQf/8KeGurn177h64y8z6aA9q/Bv4AuCkzvzShFjQOAn6YmR+r6TwOvAn4PZqzESPT35y3J1PpEeC2iBj5P6avpjkDurkay3ZkE7U8XMcT8/0E4NzM/Ep1/y7wp8DJrcGWAq+OiB37jHoox0GjbF9Polkvb2v1vywzvziRaU62Huvab9NcpXYBzb9yHXEKTTj9rxr20cz86CDTiYgn04TiN9V8prYDj9JsFwAeo1mv3zTuBk2eQY+/TwLeWssmtayeR7PsjphQrqhjgZcAx7Hx/NggM++gOan2f3p077fNHHQftitwd2ucN7fKJ7of3YgheZLUTv8wmrMb/byaZoX/FP13jF8GfhQRL5ucGo7PGNo14js0lzi8tqN8H5pfuTZ7tZLPofl1Fp44GLyYJuRsNcBoXh0Rq1qvbYdU3X7e1Jr+lRMYz0s72rJXu2NEzAEO5on/RX4h8Irq9/SIeMEYptVvOTmKZt25GvjFiNils4eI2AFYAPQ7APgK8Kwe3fqtd+NZDjbIzPU0B16HVdFRNN9VAnt1fMcv7TKKRTSXno3Ytvr9GvAPwLta3Ub9rkarLs2ZiUNpfsyb8P+Z77JedVuuxrqduAg4EngxzXzdsEOMiCcBv0ZzkDXa9nasLgCOqmk8j+bscFuv9b/XfFkKnBQRX46Id0fEgnHUqXP5APhEqw7vG8c4O/Vr93PYdN6trPIRvfYP0Hsd+Dtgb5qw/aeT0Iaudc3M71f9NhyQbs7bk2kwMu93p7kc9L86uk/W/mbbjuXg1aP0PxnbkY3U8r0/8PkqGmTZfphmPf7jPqMe2nFQl+3riOfQLKPTrT1fL+7s2GNdG1lHOrffE/kenwl8p9b3ts75+WHgdyPiKWMY92Qsuxt0Of7eaPzAO1u9D7KM9tse9a1LWQx8PjO/ATwQES/q0a5+28R+3Qfdh50BXBERl0bEmyJi+yqfjP3oRgzJE7dtLawraXaw5/TqsX6F/W5mfge4HHjBKL86vpvmjMN06Nau7NFvZ/nI2YMZv3zVWb+XA5+tjeq1NMFhNJ/OzH1brx8MtaLdndGa/kR+bLm6oy3frPKRZeRemssiL4PmzDHNZS6nAD8BLo+Igycw/RFH09zL/xPgMzThaMRLI+KrNL8irsjMe/uMp9+ZTOiy3k1gOej0KZ74BfYonjgT982O77h9xuPKiFhLs7Nsn7n7QfX7LJqdy/mtswj9vqtBjfx6367nZOq1XI3FhTRt63Y29zeBK2vd+wxweP2gM2GZeRPNZWZH0/1fDfZa/7vOl8xcRXOZ2vuAHYHrI+LZDLbNfV9EfAP4JM0l522/26rDW5mgAdo9iF77h67rQH1Xfw9cmpkPjHOaYzVTtidT6fPAr9NsDz7dpftk7W9+0LEcdJtW22RsR0bsVfu0+2iuhrppjMOfCSyJiJ+bQB2GJiJ2qrDxjYh4yxRPvj1fX9kq77qu1Q+IC4AvVSj7cUTsM1WVrfXyfHqcEe1hspbdXrlio/EDfzGWNpVeuWKQ9ehomuMC6m+vH55H2yZ2dh/TPqzO/D+b5n7tA4FrImKbPvvRcZvxIWYz0F5o/ygzf9Sn36OBZ0XEnTT3yG0H/E6vnjPzCpr7NA6YzAoPqFu7HgB26OhvR+C77YLMvJ3msplXtYpXA71+ddqsRMQzaH4pv5/mwGV74Oaaby9h8780bqr8oDbUT6fZ6G24rKcuf7q0Nmp/SfOwh0F0XU4i4rk0O8zLaj4cxcbz4erMfD7Nr6bHRcS+fabxApp7nLrqsd5N1nJwCXBwRLyQ5mEcg/wa/jKa73gVzX3J3er8ZZr75OYO8F0NJDOvA55Lc4/TN8Y6fKeO9aqXMW0n6oDqxzQH75d3dD4a+LX6Dm6geeDKQUyeZTSXAg96qXXf+ZKZD2fmP2fmG4F/pAlRg2xz35qZv0Bzud3S8TVlTHq1+1Y2nXcvopmnG/TYP4zmJ/WaLJvUNSK2A/aguY96pmxPpkwdA9xA84Coi6a5OqMZ7/HGN2ufthfwooj4rSofdNn+Hs2B/gl0N7TjoD7b19U096eSmQ9U+86mue1jc9BrXXsVzbbvW7WOzOeJdWQi3+M3gT26/JCxyfykuT3kOOBnxjmt8RpLrhgx6DI6rlxRJ/QOAv6h5sdbaeZRt0Dcd5vYpfuY92GZ+V+ZuTQzF9NcHr9PlXfbj46bIXmKRMQWNAvUczNzfmbOp7l0YbQd47uZvEvMJiQzHwbuiYiDYMNKs4jm8rlOp9Hcozbik8CLI2LkfjYi4len8pfBQUTEXJrL+z6UmUkzf36/Nc/2BH49mvtaBGTmIzS/tr45IraMiBdGxNNgw3L/PJr7RwdxBbBNRGy4XyUinkfzC/3bR+ZDZj4NeFpEPL2jLt+iecL4Sd1GHhH/i+ZemNHuX+pc7yZlOah16EqaHcHAZ2cz8zGapyIf0+3qk4h4Fs2ldg9UXUf9rgZ0MvC2cQzXWb/O9aqXrvM/ul9+PuIvgJOy7i+rYbajedjXHq15dgKTG0SW0jy8ZNBbUXrOl4j4lbrccOQs497At8e4zf0QsEVEDPuMZK92vx84JZp77UfuuX8bcHqXcXTuH6ba5cCTI+IY2HDLyOk0D5F5ZKSnzX17Mg1Op1nX1k93RUYxnu3IBtnc13kyzdVQ0Fx6+7qRABcRO9Gc8frrLoN/AHgDzdPfOw3lOGiU7etfA3/WcUZts1vOuqxrRwOLWuvIi3jiKqy/ojn7+FRotpkR8fsDTud/aO7V/cDIlUW1HXgyzXLT7nc9zdVKx02gaVPlr4H31rJJLauvAz7Spd/x5IojgI9n5tNrnuxO83Cz3ds91Xb//cAHu41klG3mQPuwiFgUdXtKLQM7AWt77UcHbmEXhuTheXJE3D3yAv4cWJv1oIHyRWDviNi110gycznN49M3F8cAf16XglxBc7C0yWUZmbma1n0w2Vxu+JvAH0XE7RFxK/BGNo+2jdznsZrmPsx/A95RByyLgH8d6bE2sF8CXjHKODvvSXzxsCpPx7IWEf+3yt/UUYf54xx/570qR3T2kJk30jwR+Wiae6I+FxG3VNljNBu/UdUO/pU0ZwC/WfPkr2guqem8j+liuj884u+AX221d2RefIPmoP13svUQkx712LDeDbgcvK5jHszrM/pP0TxptR2SO+/H3OQSr8y8p4YZOUux4f4kmssfl1RQPIrBv6u+6mqA8d5f2HW9anXfZLnqM/97Xu6amf+ZmZ33Mb0SuCI3fmjHJTT3ym9Tn/+1Nb/+aayNy+aBdGf26Nxt/e83X/YCvhARI//OZCXN5dgw+DY32fTgp30/17+PtY3d9Gp3Xep2Es26/zWae8H/tMo7+91o/1BGXQcmS2s5OzIibge+QfNE224/CG3u25Mpk5mrM/O8Hp0na3/TeV/ne1rduq2zk7Id6eKzNPvWl9a29zXAR2vZ/k9gabYewDWiAvbFNE9c7uw2mcdBo21fR6Z5M8190udHxNcj4j9oLlX95DimOWztde3pNE9+BzaE6IciYv9anz4E/Hu1/ys0V2YO6hSa9f0btf4fCbyyxw+4p9NcpTWICS+7Y2jDRjJzGc0PmP9Zy+hHgdfUstvZb7dcMVpdjmbT/ddnaL7LvSLixoi4jeZHhTOzHopYBtpmjmEfdghwSzSX6a+gORN9L/33o+MS3ZcJSZIkSZJ++ngmWZIkSZKkYkiWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRploqIt0fEdP5PYkmSZhxDsiRJkiRJxZAsSdIsERHHRMRNEfHViPh4R7fXR8T11e0zEfHkKj8yIm6p8i9W2XMi4rqIWFXjWzAd7ZEkaTpEZk53HSRJ0gRFxHOAi4EXZ+Z3I2JH4P8AD2fm+yNip8x8oPp9N3BfZn4wIm4GFmXm2ojYPjO/FxEfBK7JzE9ExNbAnMz8wXS1TZKkqeSZZEmSZoeDgH/KzO8CZOb6ju77RMTVFYp/F3hOlf8HcG5EvB6YU2VfBt4WEScBTzcgS5J+mhiSJUn66XAucGJmPhd4B/AkgMz8A+D/AbsDN9QZ508CvwX8AFgeEQdNT5UlSZp6hmRJkmaHK4AjI2IngLrcuu3ngHsiYiuaM8lUf3tl5rWZ+RfAOmD3iHgGcEdmnglcAjxvSlogSdJmYMvproAkSZq4zFwdEacBX4iIx4EbgTtbvfw5cC1NEL6WJjQDvK8ezBXA5cBXgZOA10bEj4F7gb+ckkZIkrQZ8MFdkiRJkiQVL7eWJEmSJKkYkiVJkiRJKoZkSZIkSZKKIVmSJEmSpGJIliRJkiSpGJIlSZIkSSqGZEmSJEmSiiFZkiRJkqTy/wFJ7YlJWzh7MwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "# let's see how many training tokens fall into each class by\n",
    "# plotting a count graph for the \"Class\" column of the data\n",
    "fig = plt.figure(figsize=(16,6))\n",
    "sns.countplot(x='class',data = X_train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PLAIN         367249\n",
       "PUNCT          96157\n",
       "DATE           15113\n",
       "LETTERS         7631\n",
       "CARDINAL        6850\n",
       "VERBATIM        3916\n",
       "MEASURE          782\n",
       "ORDINAL          625\n",
       "DECIMAL          516\n",
       "MONEY            309\n",
       "DIGIT            298\n",
       "ELECTRONIC       217\n",
       "TELEPHONE        189\n",
       "TIME              61\n",
       "FRACTION          56\n",
       "ADDRESS           31\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the counts for number of sample tokens in each class group\n",
    "X_train_data['class'].value_counts().sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Create the input and target vocabularies\n",
    "\n",
    "These are the vocabularies that the model can draw input from and decode output to."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "347\n",
      "Vocab created in 7.653698921203613 s.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# create vocabularies\n",
    "# target vocab\n",
    "y = list(np.where(X_train_data['class'] == \"PUNCT\", \"sil.\",\n",
    "      np.where(X_train_data['before'] == X_train_data['after'], \"<self>\",\n",
    "               X_train_data['after'])))\n",
    "\n",
    "y = [token.split() for token in y]\n",
    "dist = FreqDist(np.hstack(y))\n",
    "temp = dist.most_common(target_vocab_size-1)\n",
    "temp = [word[0] for word in temp]\n",
    "temp.insert(0, 'ZERO')\n",
    "temp.append('UNK')\n",
    "\n",
    "target_vocab = {word:ix for ix, word in enumerate(temp)}\n",
    "print(len(target_vocab))\n",
    "target_vocab_reversed = {ix:word for word,ix in target_vocab.items()}\n",
    "\n",
    "# input vocab\n",
    "X = list(X_train_data['before'])\n",
    "X = [list(token) for token in X]\n",
    "\n",
    "dist = FreqDist(np.hstack(X))\n",
    "temp = dist.most_common(input_vocab_size-1)\n",
    "temp = [char[0] for char in temp]\n",
    "temp.insert(0, 'ZERO')\n",
    "temp.append('<norm>')\n",
    "temp.append('UNK')\n",
    "\n",
    "input_vocab = {char:ix for ix, char in enumerate(temp)}\n",
    "\n",
    "gc.collect()\n",
    "\n",
    "print('Vocab created in {0} s.'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Replaced tokens with integers in 1.0112733840942383 s.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Converting input and target tokens to index values\n",
    "X = index(X, input_vocab)\n",
    "y = index(y, target_vocab)\n",
    "\n",
    "gc.collect()\n",
    "\n",
    "print('Replaced tokens with integers in {0} s.'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Data preprocessing\n",
    "\n",
    "Add the context window (3 tokens previous and 3 tokens in the future) for each token of the training data. Pad these sequences to make each input sequence have a fixed length, and convert to an"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added context window to X in 2.1028590202331543 s.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# adding a context window of 3 words in input, with token separated by <norm>\n",
    "X = add_context_window(X, context_size, padding_entity, input_vocab)\n",
    "\n",
    "print('Added context window to X in {0} s.'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added padding and converted to np array in 36.42306995391846 s.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# padding\n",
    "X = padding_batchwise(X, X_seq_len)\n",
    "y = padding_batchwise(y, y_seq_len)\n",
    "\n",
    "# convert to integer array, batch-wise (converting full data to array at once takes a lot of time)\n",
    "X = np.array(X)\n",
    "y = np.array(y)\n",
    "y_sequences = np.asarray(sequences(y, y_seq_len, target_vocab))\n",
    "\n",
    "print('Added padding and converted to np array in {0} s.'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Training the LSTM encoder/decoder\n",
    "\n",
    "I'll train 500,000 sample tokens for 5 epochs, with both model checkpointing (saving the model) after each epoch and early stop checking (stop training early if the validation accuracy gets worse- this means the model overfit!)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting model...\n",
      "Train on 450000 samples, validate on 50000 samples\n",
      "Epoch 1/5\n",
      "450000/450000 [==============================] - 10409s 23ms/step - loss: 0.1299 - acc: 0.9722 - val_loss: 0.0472 - val_acc: 0.9899\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04722, saving model to saved_model.hdf5\n",
      "Epoch 2/5\n",
      "450000/450000 [==============================] - 16693s 37ms/step - loss: 0.0374 - acc: 0.9914 - val_loss: 0.0321 - val_acc: 0.9924\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.04722 to 0.03211, saving model to saved_model.hdf5\n",
      "Epoch 3/5\n",
      "450000/450000 [==============================] - 11865s 26ms/step - loss: 0.0281 - acc: 0.9931 - val_loss: 0.0259 - val_acc: 0.9934\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.03211 to 0.02590, saving model to saved_model.hdf5\n",
      "Epoch 4/5\n",
      "450000/450000 [==============================] - 10653s 24ms/step - loss: 0.0236 - acc: 0.9941 - val_loss: 0.0220 - val_acc: 0.9943\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.02590 to 0.02203, saving model to saved_model.hdf5\n",
      "Epoch 5/5\n",
      "450000/450000 [==============================] - 10224s 23ms/step - loss: 0.0209 - acc: 0.9949 - val_loss: 0.0206 - val_acc: 0.9951\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.02203 to 0.02060, saving model to saved_model.hdf5\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "\n",
    "# fitting the model on the validation data with batch size set to 128 for a total of 5 epochs:\n",
    "print('Fitting model...')\n",
    "checkpointer = ModelCheckpoint(filepath='saved_model.hdf5', verbose=1, save_best_only=True)\n",
    "earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=1, mode='auto')\n",
    "callbacks_list = [checkpointer, earlystop]\n",
    "\n",
    "history = model.fit(X, y_sequences, batch_size=batch_size, epochs=epochs, validation_split=val_split, callbacks=callbacks_list, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcVPWV///X6X1vmqbZl+4GF1bZBBQhLolRo6hxAXeMQuJ3Mk5+SSYhmZnoZJKZ5JvE+M3EGHGLuxKM0USNiXEBjAuIyiIi+7420Gzd0E2f3x/3NhRtLwV0dVV3v5+PRz361r2fe+vUhapTn8+991xzd0RERBqTFO8AREQk8SlZiIhIk5QsRESkSUoWIiLSJCULERFpkpKFiIg0SclCYsrMis3MzSwliraTzWxOjONZbGZnx/I1YiXcj/3C6d+a2X9E0/Y4Xuc6M/vr8cbZyHbPNrP1zb1daRlKFnKYma02s4Nm1qnO/A/CL5/i+ER2bEmnMe4+0N3faKaw4sbdv+bu/3Wi26lvv7r7E+5+/oluW9oWJQupaxVwTe0TMxsMZMUvnOidaCIRkYYpWUhdjwE3Rjy/CXg0soGZ5ZvZo2a2zczWmNm/m1lSuCzZzH5uZtvNbCXwpXrWfdDMNpnZBjP7kZklRxHXrPDvLjPba2ZnhMNWb5nZL82sDLjTzPqa2WtmVhbG8ISZdYh4/dVm9vlw+k4zmxG+lz3hENXI+l7czO41s5/Xmfe8mX0znP5u+H72mNlSMzuvnm2MNrPNke/XzC43swXh9Cgze9vMdoX759dmltZAPL8zsx9FPP/XcJ2NZvaVOm2/FPYOd5vZOjO7M4r9Oidi/TPNbK6ZlYd/z4xY9oaZ/Vf477DHzP5at2faEDPrH66/K9z3EyKWXWRmH4fb3GBm3w7ndzKzP4fr7DCz2bX/9yTG3F0PPXB3gNXA54GlQH8gGVgP9AEcKA7bPQo8D+QCxcCnwC3hsq8BnwC9gI7A6+G6KeHy54D7gGygM/Ae8NVw2WRgTgOxFUduJ6J9NfDPQAqQCfQDvgCkA0UEX4Z3132P4fSdQCVwUfhe/wd4p4HXHw+sAyx8XgBUAN2BU8Jl3SNi7dvAdlYAX4h4/ntgWjg9AhgTvpdiYAnwjYi2DvQLp38H/CicvgDYAgwK9+uTddqeDQwm+HE4JGx7WRP7dU443RHYCdwQxnVN+LwwXP5G+J5ODvf/G8BPGnjvZwPrw+lUYDnwfSANOBfYA5wSLt8EjIvY18PD6f8BfhuunwqMq/030SO2D2VkqU9t7+ILBF9YG2oXhL+KJwHfc/c97r4a+AXBlwnA1QRfzuvcfQfBh7t23S4EX8zfcPd97r4V+GW4veO10d3/192r3b3C3Ze7+9/c/YC7bwPuAj7XyPpz3P0ldz8Uvu/TGmg3m+BLdVz4/ErgbXffCBwiSE4DzCzV3Ve7+4oGtvMU4TCfmeUS7I+nANz9fXd/J3wvqwmSamOx17oaeNjdF7n7PoIkeJi7v+HuC929xt0XhK8XzXYh6Bkuc/fHwrieIvgxcElEm4fd/VN3rwBmAEOj2O4YIIcgsRx099eAP3NkCLSKYH/muftOd58fMb8b0Mfdq9x9trurwF0LULKQ+jwGXEvwC/PROss6EfyiWxMxbw3QI5zuTvArO3JZrT7hupvCYYRdBF+InU8g1sjXwsy6mNnT4dDFbuDxMOaGbI6Y3g9k1HfsI/xCepojX2bXAk+Ey5YD3yD4kt4avn73Bl7vSeDLZpYOfBmY7+5rwthPDodYNoex/3cTsddqbJ/XDn+9Hg4blhP0/qIaKgq3vabOvMh/b/jsPsyJNmZ3r2lgu1cQJNI1ZvammZ0Rzv8ZQY/kr2a20symRfc25EQpWchnhF9eqwg+rH+os3g7wa+7PhHzenOk97GJYAgqclmtdcABoJO7dwgfee4+MJqwopz/3+G8we6eB1wPWBTbj8ZTwJVm1gcYDTx7OAj3J939LI4M2f203mDdPyb4UryQIOE8GbH4XoJf7SeFsX8/ytgb2+eEr/EC0Mvd8wmGcWq329Sv8o0c/W9du/0N9bQ9FhuBXnWONxzerrvPdfdLCX5I/JGgx0LYm/2Wu5cCE4Bv1nd8SJqfkoU05Bbg3HBY47BwuGYG8GMzyw2/OL9J8AuecNntZtbTzAqAaRHrbgL+CvzCzPLMLCk8IB3NkMg2oAYobaJdLrAXKDezHsC/RrHtqLj7BwTJ8gHgFXffBWBmp5jZuWFvoZLgWEZNw1viSeBfCI6D/L5O7LuBvWZ2KnBblKHNACab2QAzywLuqLM8F9jh7pVmNoogSdVqar++BJxsZteaWYqZTQQGEAwZnYh3CXoh3zGzVAuufbkEeNrM0iy41iPf3asI9kkNgJldbGb9zMyAcoIhwMb2tTQTJQupl7uvcPd5DSz+Z2AfsBKYQ/Dl91C47H7gFeAjYD6f7ZncSHBA82OCA6UzCcagm4pnP/Bj4K1wCGtMA03/ExhO8EXyYj2vf6KeJDgJILJHkA78hCCRbCb4Nfy9RrZRe8zgNXffHjH/2wRf5HsI9uMz0QTk7i8DdwOvEQzRvFanyf8Bfmhme4AfEP5KD9dtdL+6exlwMfAtoAz4DnBxnbiPmbsfJEgOFxLst98AN7r7J2GTG4DV4XDc14DrwvknAa8S/CB4G/iNu79+IrFIdGrP7BAREWmQehYiItIkJQsREWmSkoWIiDRJyUJERJrUZgqvderUyYuLi+MdhohIq/L+++9vd/eiptq1mWRRXFzMvHkNnekpIiL1MbO6V+jXS8NQIiLSJCULERFpkpKFiIg0qc0csxCRtqWqqor169dTWVkZ71DahIyMDHr27Elqaupxra9kISIJaf369eTm5lJcXExQN1COl7tTVlbG+vXrKSkpOa5taBhKRBJSZWUlhYWFShTNwMwoLCw8oV6akoWIJCwliuZzovuy3SeL8ooq7vrrUpZv3RPvUEREEla7TxaHapz7Zq3kgdmr4h2KiCSQXbt28Zvf/OaY17vooovYtWtXDCKKr3afLDpmp3HVyJ78Yf4Gtu7RWRciEmgoWVRXVze63ksvvUSHDh1iFVbctPtkAXDLWaVU1dTw6D+iuupdRNqBadOmsWLFCoYOHcrpp5/OuHHjmDBhAgMGDADgsssuY8SIEQwcOJDp06cfXq+4uJjt27ezevVq+vfvz5QpUxg4cCDnn38+FRUV8Xo7J0ynzgIlnbL54oCuPPbOGm47uy/Z6dotIonkP/+0mI837m7WbQ7onscdlwxscPlPfvITFi1axIcffsgbb7zBl770JRYtWnT41NOHHnqIjh07UlFRwemnn84VV1xBYWHhUdtYtmwZTz31FPfffz9XX301zz77LNdff32zvo+Wop5FaMr4Usorqvj9vHXxDkVEEtCoUaOOukbhV7/6Faeddhpjxoxh3bp1LFu27DPrlJSUMHToUABGjBjB6tWrWyrcZqef0KERfQoY2aeAB+as4voxfUhJVh4VSRSN9QBaSnZ29uHpN954g1dffZW3336brKwszj777HqvYUhPTz88nZyc3KqHofSNGGHK+FLW76zgL4s3xzsUEYmz3Nxc9uyp/5T68vJyCgoKyMrK4pNPPuGdd95p4ehannoWEb7QvwslnbKZPmslXxrcTRcEibRjhYWFjB07lkGDBpGZmUmXLl0OL7vgggv47W9/S//+/TnllFMYM2ZMHCNtGebu8Y6hWYwcOdKb4+ZHT7y7hn97bhFPTx3DmNLCplcQkZhYsmQJ/fv3j3cYbUp9+9TM3nf3kU2tq2GoOq4Y3pPC7DSmz1oZ71BERBKGkkUdGanJ3HhGMa99spVlW1QCREQElCzqdcMZfchITeL+2epdiIiAkkW9OmancdWIXvzxg41s3a0SICIiShYNuHVcCVU1NfzuH6vjHYqISNwpWTSgT2E2FwzsyuPvrGHfgcYLh4mItHUxTRZmdoGZLTWz5WY2rZ7l481svplVm9mVEfOHmtnbZrbYzBaY2cRYxtmQqeNL2V1ZzTNzVQJERBqXk5MDwMaNG7nyyivrbXP22WfT1Cn+d999N/v37z/8PFFKnscsWZhZMnAPcCEwALjGzAbUabYWmAw8WWf+fuBGdx8IXADcbWYtXvN3WO8CTi8u4ME5q6g+VNPSLy8irVD37t2ZOXPmca9fN1kkSsnzWPYsRgHL3X2lux8EngYujWzg7qvdfQFQU2f+p+6+LJzeCGwFimIYa4Omju/Lhl0VvLRIJUBE2pNp06Zxzz33HH5+55138qMf/YjzzjuP4cOHM3jwYJ5//vnPrLd69WoGDRoEQEVFBZMmTaJ///5cfvnlR9WGuu222xg5ciQDBw7kjjvuAILihBs3buScc87hnHPOAY6UPAe46667GDRoEIMGDeLuu+8+/HotUQo9luU+egCR4zfrgdHHuhEzGwWkASvqWTYVmArQu3fv44uyCeed2pnSomymz1rBJUNUAkQkLl6eBpsXNu82uw6GC3/S4OKJEyfyjW98g3/6p38CYMaMGbzyyivcfvvt5OXlsX37dsaMGcOECRMa/F649957ycrKYsmSJSxYsIDhw4cfXvbjH/+Yjh07cujQIc477zwWLFjA7bffzl133cXrr79Op06djtrW+++/z8MPP8y7776LuzN69Gg+97nPUVBQ0CKl0BP6ALeZdQMeA25298+MA7n7dHcf6e4ji4pi0/FISjKmjCtl0YbdvL2yLCavISKJZ9iwYWzdupWNGzfy0UcfUVBQQNeuXfn+97/PkCFD+PznP8+GDRvYsmVLg9uYNWvW4S/tIUOGMGTIkMPLZsyYwfDhwxk2bBiLFy/m448/bjSeOXPmcPnll5OdnU1OTg5f/vKXmT17NtAypdBj2bPYAPSKeN4znBcVM8sDXgT+zd3jWtLx8mE9+MVfl3L/rJWc2bdT0yuISPNqpAcQS1dddRUzZ85k8+bNTJw4kSeeeIJt27bx/vvvk5qaSnFxcb2lyZuyatUqfv7znzN37lwKCgqYPHnycW2nVkuUQo9lz2IucJKZlZhZGjAJeCGaFcP2zwGPuvvxHylqJhmpydx0RjGvL93GpyoBItJuTJw4kaeffpqZM2dy1VVXUV5eTufOnUlNTeX1119nzZrGb8U8fvx4nnwyOH9n0aJFLFiwAIDdu3eTnZ1Nfn4+W7Zs4eWXXz68TkOl0ceNG8cf//hH9u/fz759+3juuecYN25cM77bxsUsWbh7NfB14BVgCTDD3Reb2Q/NbAKAmZ1uZuuBq4D7zGxxuPrVwHhgspl9GD6GxirWaFw/pg+ZqcncrwKDIu3GwIED2bNnDz169KBbt25cd911zJs3j8GDB/Poo49y6qmnNrr+bbfdxt69e+nfvz8/+MEPGDFiBACnnXYaw4YN49RTT+Xaa69l7Nixh9eZOnUqF1xwweED3LWGDx/O5MmTGTVqFKNHj+bWW29l2LBhzf+mG6AS5cfgjucX8eR7a3nru+fSOS8jpq8l0t6pRHnzU4nyFnLLWaUcqnEeVgkQEWlnlCyOQe/CLC4c1I0n3lnDXpUAEZF2RMniGE1RCRCRFtNWhskTwYnuSyWLYzS0VwdGlXTkIZUAEYmpjIwMysrKlDCagbtTVlZGRsbxH2uN5XUWbdbUcaXc+ug8Xly4iUuH9oh3OCJtUs+ePVm/fj3btm2LdyhtQkZGBj179jzu9ZUsjsO5p3amb1E2989eyYTTuqsEiEgMpKamUlJSEu8wJKRhqONwVAmQFSoBIiJtn5LFcbpsWA865aQzXffpFpF2QMniOGWkJjP5zD68sXQbSzerBIiItG1KFifgutFhCRD1LkSkjVOyOAEF2WlMPL0Xz3+4gc3lx18xUkQk0SlZnKBbzirhUI3zO5UAEZE2TMniBPXqmMWFg7vxxLsqASIibZeSRTOYOq6UPZXVPP3e2niHIiISE0oWzeC0Xh0YHZYAqVIJEBFpg5QsmsnU8aVsLK/kpYWb4h2KiEizU7JoJuec0pl+nXO4782VKnwmIm2OkkUzCUqAlPDxpt38QyVARKSNUbJoRrUlQO7TfbpFpI1RsmhG6SnJ3Dy2mFmfbmPJpt3xDkdEpNkoWTSz60b3JitNJUBEpG1RsmhmHbLSuHpkL174cCObyiviHY6ISLNQsoiBW84qocad3721Ot6hiIg0CyWLGOjVMYuLBnfjyXfXsqeyKt7hiIicMCWLGJk6vpQ9B6p5+r118Q5FROSEKVnEyJCeHRhT2pGH3lIJEBFp/ZQsYuir4/uyqbySFxeoBIiItG4xTRZmdoGZLTWz5WY2rZ7l481svplVm9mVdZbdZGbLwsdNsYwzVj53chEndc7hvlkqASIirVvMkoWZJQP3ABcCA4BrzGxAnWZrgcnAk3XW7QjcAYwGRgF3mFlBrGKNlaQkY8r4UpZs2s1by1UCRERar1j2LEYBy919pbsfBJ4GLo1s4O6r3X0BUHdQ/4vA39x9h7vvBP4GXBDDWGPm0qHd6Zybzn2zVsQ7FBGR4xbLZNEDiDwVaH04r9nWNbOpZjbPzOZt27btuAONpfSUZCaPLWb2su0qASIirVarPsDt7tPdfaS7jywqKop3OA26blSfoASICgyKSCsVy2SxAegV8bxnOC/W6yac/KxUJp3emxc+UgkQEWmdYpks5gInmVmJmaUBk4AXolz3FeB8MysID2yfH85rtW4eW4wDD6sEiIi0QjFLFu5eDXyd4Et+CTDD3Reb2Q/NbAKAmZ1uZuuBq4D7zGxxuO4O4L8IEs5c4IfhvFYrsgTIbpUAEZFWxtrK+f8jR470efPmxTuMRi1cX84lv57D9y86lanj+8Y7HBERzOx9dx/ZVLtWfYC7tRncM58zSgt5aM5qDlarBIiItB5KFi1s6udK2by7kj8v2BjvUEREoqZk0cLOPrmIk7vkMF0lQESkFVGyaGFmxpRxpXyyeQ+zl22PdzgiIlFRsoiDCWEJEN2nW0RaCyWLOEhPSebmsSXMXradxRvL4x2OiEiTlCzi5NrRvclOS+aB2aviHYqISJOULOIkPzOVSaN686ePNrJxl0qAiEhiU7KIoyMlQNS7EJHEpmQRRz0Lsrh4SDeeem+dSoCISEJTsoizKeNK2XugmqfeXRvvUEREGqRkEWeDeuQztl8hD7+lEiAikriULBLAlHFBCZA/faQSICKSmJQsEsDnTi7ilC653D9bJUBEJDEpWSQAM2PK+KAEyCyVABGRBKRkkSAmnNadLnnpTJ+1It6hiIh8hpJFgkhLSeLmsSW8tbyMRRtUAkREEouSRQK5dnRvctJTeEAFBkUkwShZJJC8jFQmnd6LPy3YxAaVABGRBKJkkWC+clYJBjw8RyVARCRxKFkkmO4dMsMSIGspr1AJEBFJDEoWCWjK+FL2HTzEU++pBIiIJAYliwQ0sHs+Z/XrxMNvrVIJEBFJCEoWCWrq+FK27D7ACyoBIiIJQMkiQY07qROnds3l/lkqASIi8adkkaDMjKnjS1m6ZQ9vfrot3uGISDunZJHALh7Sna55GUyfpYv0RCS+YposzOwCM1tqZsvNbFo9y9PN7Jlw+btmVhzOTzWzR8xsoZktMbPvxTLORBWUACnmHytUAkRE4itmycLMkoF7gAuBAcA1ZjagTrNbgJ3u3g/4JfDTcP5VQLq7DwZGAF+tTSTtzTVhCRD1LkQknmLZsxgFLHf3le5+EHgauLROm0uBR8LpmcB5ZmaAA9lmlgJkAgeB3TGMNWHlZaRyzahevLhwE+t37o93OCLSTsUyWfQA1kU8Xx/Oq7eNu1cD5UAhQeLYB2wC1gI/d/cdMYw1od08NigB8tCc1fEORUTaqUQ9wD0KOAR0B0qAb5lZad1GZjbVzOaZ2bxt29ruGUPdO2RyyWndeXruWsr3qwSIiLS8WCaLDUCviOc9w3n1tgmHnPKBMuBa4C/uXuXuW4G3gJF1X8Ddp7v7SHcfWVRUFIO3kDimjCtl/8FDPPHemniHIiLtUFTJwsz+xczyLPCgmc03s/ObWG0ucJKZlZhZGjAJeKFOmxeAm8LpK4HXPLgCbS1wbvja2cAY4JPo3lLbNKB7HuNO6sTv3lrNgepD8Q5HRNqZaHsWX3H33cD5QAFwA/CTxlYIj0F8HXgFWALMcPfFZvZDM5sQNnsQKDSz5cA3gdrTa+8BcsxsMUHSedjdFxzD+2qTpo4vZeueAzz/oUqAiEjLSomynYV/LwIeC7/0rbEVANz9JeClOvN+EDFdSXCabN319tY3v707q18n+nfL4/5ZK7lqRE+i+CcQEWkW0fYs3jezvxIki1fMLBdQOdQWFpQAKWHZ1r28sbTtHtAXkcQTbbK4hWCI6HR33w+kAjfHLCpp0MVDutMtXyVARKRlRZsszgCWuvsuM7se+HeCayKkhaUmJ/GVsSW8vbKMhev1TyAiLSPaZHEvsN/MTgO+BawAHo1ZVNKoSaN6kZuewvTZ6l2ISMuINllUh6e0Xgr82t3vAXJjF5Y0JjcjlWtH9+alhZtYt0MlQEQk9qJNFnvCyq83AC+aWRLBcQuJk8lji4MSIG+tincoItIORJssJgIHCK632ExwNfbPYhaVNKlbfiYThnbnmbnrVAJERGIuqmQRJogngHwzuxiodHcds4iz2hIgj7+rEiAiElvRlvu4GniP4EK5q4F3zezKWAYmTevfLY/xJxfxu3+oBIiIxFa0w1D/RnCNxU3ufiNBVdj/iF1YEq2p40rZtucAz3+gEiAiEjvRJouksPprrbJjWFdiaGy/QgZ0y2P67JXU1Hi8wxGRNiraL/y/mNkrZjbZzCYDL1Kn5pPER1ACpJTlW/fyxqdbm15BROQ4RHuA+1+B6cCQ8DHd3b8by8Akel8a0o3uKgEiIjEUbdVZ3P1Z4NkYxiLHKTU5ia+cVcKPXlzCgvW7GNKzQ7xDEpE2ptGehZntMbPd9Tz2mNnulgpSmjZpVG9yM1LUuxCRmGi0Z+HuKunRSuSkp3Dt6N7cP2sl63bsp1fHrHiHJCJtiM5oakNuPrOE5CTjwTkqASIizUvJog3pmp/BhNN68MzcdezafzDe4YhIG6Jk0cZMGV9CRdUhnnh3bbxDEZE2RMmijTm1ax6fO7mIh99aTWWVSoCISPNQsmiDpo4vZfveAzz/4YZ4hyIibYSSRRt0Zt9CBnbPY/oslQARkeahZNEG1ZYAWbFtH68vVQkQETlxShZt1EWDu9GjQyb36SI9EWkGShZtVGpyEjePLea9VTv4cN2ueIcjIq2ckkUbVlsC5H71LkTkBClZtGE56SlcN7oPLy/axNqy/fEOR0RaMSWLNu7mscVhCRD1LkTk+MU0WZjZBWa21MyWm9m0epanm9kz4fJ3zaw4YtkQM3vbzBab2UIzy4hlrG1Vl7wMLh3agxnz1rNzn0qAiMjxiVmyMLNk4B7gQmAAcI2ZDajT7BZgp7v3A34J/DRcNwV4HPiauw8EzgaqYhVrWzd1fCkVVYd4/J018Q5FRFqpWPYsRgHL3X2lux8EngYurdPmUuCRcHomcJ6ZGXA+sMDdPwJw9zJ3V+2K43Ryl1zOPqWIR95WCRAROT6xTBY9gHURz9eH8+pt4+7VQDlQCJwMeHjf7/lm9p36XsDMpprZPDObt23btmZ/A21JUALkIM99oBIgInLsEvUAdwpwFnBd+PdyMzuvbiN3n+7uI919ZFFRUUvH2KqcUVrIoB553D9bJUBE5NjFMllsAHpFPO8Zzqu3TXicIh8oI+iFzHL37e6+H3gJGB7DWNu8oARIX1Zu28ffP1EJEBE5NrFMFnOBk8ysxMzSgEnAC3XavADcFE5fCbzm7g68Agw2s6wwiXwO+DiGsbYLFw3qSo8OmbpIT0SOWcySRXgM4usEX/xLgBnuvtjMfmhmE8JmDwKFZrYc+CYwLVx3J3AXQcL5EJjv7i/GKtb2IiU5iVvOKuG91Tv4YO3OeIcjIq2IBT/kW7+RI0f6vHnz4h1Gwtt3oJoz/ufvnHVSJ35z3Yh4hyMicWZm77v7yKbaJeoBbomR7PQUrh/Th78s2syasn3xDkdEWgkli3Zo8pnFpCQl8eCcVfEORURaCSWLdqhzXgaXDevOjHnrVAJERKKiZNFOTRlXSmVVDY+pBIiIREHJop06qUsu557amUf+oRIgItI0JYt2bMq4Usr2HeQP81UCREQap2TRjo0p7ciQnvk8oBIgItIEJYt2zMyYMq6Uldv38eqSLfEOR0QSmJJFO3fhoK70LMjk/tkqASIiDVOyaOdqS4DMXb2T+SoBIiINULIQrh7Zi/zMVBUYFJEGKVlIWAKkN39ZvJnV21UCREQ+S8lCALjpjGJSVQJERBqgZCFAUALk8mE9+P3769ihEiAiUoeShRw2ZXxJUALkbZUAEZGjKVkAVO6OdwQJoV/nXM47tTOPvq0SICJyNCWLil3wi1PgsS/DR8/Agb3xjiiupowPSoA8O399vEMRkQSiZOE1MOY22L4MnpsKPz8Jnp0Cy16FQ9Xxjq7FjS7pyGk983lg9ioOqQSIiISULLI6wnk/gH/5CG5+GYZMhGV/hSeugLtOhZenwYb50EZuP9sUM2PK+FJWqQSIiETQPbjrU30Alv0NFjwDn/4FDh2EwpOCRDL4SuhY0jyvk6CqD9Vwzi/eoHNuBs/edma8wxGRGNI9uE9ESjr0vxgmPgbf/hQu+RXkdIHXfwS/GgoPng9zH4D9O+IdaUykJCdxy9gS3l+zk/fXtM33KCLHRsmiKZkFMOImuPlF+MZCOO8OqCyHF78FPz8ZnroGFj8HVRXxjrRZXX16UAJkukqAiAiQEu8AWpUOvWHcN+Gs/w82LwyGqRbOhKUvQXoeDJgQDFX1OQuSWncezkpL4YYxfbjnjeWs2r6Pkk7Z8Q5JROKodX+jxYsZdBsCX/wxfPNjuOGPcOrFsPiP8MglcPcg+NsPYMvieEd6Qm46MygB8oDKl4u0e0oWJyopGfqeA5ffC99eBlc8CF0GwT9+DfeeCfeOhbf+H5S3vluXFuWm8+XhPZj5/nrK9h6IdzgiEkdKFs0pLSs4W+q6GcGB8Qt/Bik6aNCoAAATNElEQVQZQS/jlwODXscHjwfHPFqJW8eVcqC6hkdVAkSkXdOpsy2hbAUsmBEc49i5Kkggp1wYHN/oex6kpMU7wkbd+shc5q/dxVvfPZfMtOR4hyMizSghTp01swvMbKmZLTezafUsTzezZ8Ll75pZcZ3lvc1sr5l9O5ZxxlxhXzjne3D7B3DLqzD8Rlg1C56aFJQaefFbsO69hL3wb+r4vuzYd5CZKgEi0m7FLFmYWTJwD3AhMAC4xswG1Gl2C7DT3fsBvwR+Wmf5XcDLsYqxxZlBr9Phop/Bt5bCtTOC4x0fPA4PfiG4huP1/4bty+Md6VFOLy7gtF4deHD2SpUAEWmnYtmzGAUsd/eV7n4QeBq4tE6bS4FHwumZwHlmZgBmdhmwCmjdpxQ1JDkVTv4iXPlQcGD8snuhoBje/L/w6xEw/Rx49z7Yuy3ekWJmfHV8KavL9vO3jzfHOxwRiYNYJosewLqI5+vDefW2cfdqoBwoNLMc4LvAfzb2AmY21czmmdm8bdvi/6V63DLyYOi1cOPzwam45/8Iaqrg5e8Ew1SPXwkLfg8H43fL0y8O7Ervjlm6SE+knUrUs6HuBH7p7o3WC3f36e4+0t1HFhUVtUxksZbXHc78Z/jaHLjtbRh7O2xdAn+4FX52Evzhq7D87y1eETc5ybh1XAnz1+5SCRCRdiiWyWID0Cviec9wXr1tzCwFyAfKgNHA/zWz1cA3gO+b2ddjGGti6jIAPn9nUGZk8osw+ApY+jI8/mX45QD4y/dh44ctdmD8yhE96ZCVyn1vqnch0t7EstzHXOAkMyshSAqTgGvrtHkBuAl4G7gSeM2Dc3nH1TYwszuBve7+6xjGmtiSkqD4rOBx4c9g2SvBqbjvTYd37oFOp8CQq2HwVVDQJ2ZhZKWlcOOYPvzv68v5YO1OhvbqQHiISUTauJheZ2FmFwF3A8nAQ+7+YzP7ITDP3V8wswzgMWAYsAOY5O4r62zjToJk8fPGXiuhr7OIlf074OM/Bolj7dvBvN5nBolj4GVBEcRmtm3PAc7+2evsO3iITjlpDO1VwPA+HRjeu4AhPfPJSlO5MZHWJNrrLHRRXluxcw0s/H1w4d/2TyE5DU46P7jw7+QvBmXXm8m6HfuZtWwb89fs4oO1O1m5PTjwnpxk9O+Wy7CIBNK7Y5Z6HyIJTMmivXKHTR8GZ08t/D3s2woZ+TDgsiBx9D6j2Svi7tx3kA/W7WT+ml3MX7uTj9btYt/BQwDqfYgkOCULCc6YWvVmMEy15E9QtQ/yewXHNoZcDZ37x+Zla5xPt+xh/tqd9fY+Tu2ay/De6n2IJAIlCznawX3wyUvBMNWK18APQdfBMGQSDLoC8rrF9OWj7X0M61XAab3U+xBpKUoW0rC9W2HRH4LEsXE+WBKUjA+GqfpfAum5MQ9BvQ+RxKBkIdHZvuxIRdxdayAlE069KKyIe25QlqSFRPY+Pli3kw/XHul9FGanMax3AcN6B8lDvQ+R5qFkIcfGPah8u+AZWPwHqNgJWYXBENWQidBjRFAIsQWp9yESe0oWcvyqD8LyV4PEsfRlOHQAOpYGSWPwVUHJ9ThpuvfRgWG9C9T7EImSkoU0j8py+PgFWDgDVs0GHHqeHpyCW9jvyCOnc4v3PODYeh/DehXQp1C9D5FIShbS/Mo3wKKZsOjZoLjhoYNHlqXlQmHp0QmkY9+gF5LZoUXDVO9DJHpKFhJbNYegfF1wy9iyFVC2PHjsWAG71oLXHGmb1SlMIGHyOJxMSiE1M+ahRtP7qD1wPry3eh/SvihZSPxUH4Cdq8MEsuLov3vr3Dwpr2dEAolIJB16x/RMrNrexwdrg+s+Gut9DOmZT3a6eh/SNilZSGI6sAd2rIxIILXJZFlwfKRWUgp06BMxrBXRK8nt3uwlS5rqfZzSJffwWVfqfUhbomQhrYt7UEV3R8SQVtlyKAsTS3XFkbYpmUHi6FjnGElh3+B032b6Et+57yAfrgt6Ho31Pob17sBpPTuo9yGtkpKFtB01NbBn05FjIpFDWztXQU3EXQMz8j97gL02kZzglemN9T6SDE7tmqfeh7Q6ShbSPhyqDq48L1tRp1eyAsrXAxH/v3O6HH1spGPt35LjLuHeWO+jY3Yaw3p1YHifAvp1zqFrXgbd8jMozEknOUlJRBKDkoVIVQXsWHX0mVq1vZJ92460s6SgGm/kAfbCvkEy6dAbkpKjfsnGeh+1kpOMzrnpdM3PoGteRr1/u+RlkJEa/euKHC8lC5HGVJbXf9pv2Qo4sPtIu+Q0KCgJE0idYyQ5XaI6PrJr/0HW7ahg8+5KNpcHfzeVV7JldyWby4NHbW8kUkFWKl3zM+maV5tYMumanx7OC5JKXkaKhrrkhESbLHRETtqnjHzoMTx4RHIPeh1HnfYbTi9/NSh9Uistp85B9ojjIxG3tO2QlUaHrDQGk99gOHsqq4LEUZtIwunaeQvWl1O27+Bn1stMTT66Z1JPL6WThr2kGahnIRKtmkOwe0M9p/0uD46bHHUhYuGRYyKFfYNhrpyioDeS3TlYfoyn/x6oPsTW3QeOJJE6CWVz2Fuprjn6M61hL2mMhqFEWlL1wfBA+/KjeyNlK2DPxs+2t2TI7hTU1MruHCSRnM7howtkFx2Zl1kQ9enANTVO2b6DEQmkdvjrAJt3V5zYsFdeBnmZGvZqazQMJdKSUtKg00nBo66D+2DPZti7Jbjx1N6twb3R926BvduCv9uWBvMOfXaoiaTUMHl0rpNQ6j4vIikjn6LcdIpy05sc9toSDnnV10tZuKGc7XujH/bqEp7ppWGvtkvJQiTW0rKPXIHeGHeo3HUkgezbeiS5RCaYzQuD4yqR15fUSk5vuIdyuBfTmdyczuQW5dCvc8PXnjQ47BU+f2/VjqiGvSITiYa9Wi8lC5FEYRYMOWUWQNHJjbetqQluUFW3h3I4wWyBXetg/TzYv/3o4ym1UrOOSiB1E0x6Tmd65XSmV/fOUNyxgTDqDHvVDn2Fw16fbtnDrE+3NTjsdXQiCYa9uuQFvZP8zFTyMlLJzUghST2VuFOyEGmNkpIguzB4dO7feNuaQ7C/7EgS2bft6CGxvVuCel1r3w7a1Sct9+hhsPA4S1JOEUU5XSjK7szg7p3h5K71XuDY0LBX7byGhr0gyKE56SmHk0d+Zip5mXWfNzxfPZjmoWQh0tYlJR/5kmdQ420PVcG+7XWGwLZEDINtha2fwN43gyGz+mTkf+aYSm5OZ3KzO9Mvpwt0L4KTu0B28VGVhSOHvXbsO0h5RRW7K6rYXVkd/K2oCuZVVrFq+z52V1RTXlFFRdVney2R0lKSwuSRQl40CSdieW5Gqo6/hJQsROSI5FTI6xY8mlJ9IOylRCSVusdZNi8I/kZe6Bgps2N4TKWI9Jwu9MoOhr7IyIeUDOiYERSOTEkP7n2SkgEpuZB6ZP5BS2d3lbG7sjpMJtWHE01tcgmSTTW7K6vYuf8ga8r2HW57qKbxM0Jz08MkkxkmnM8klxTys4IkUzfhZKQmtZmzx5QsROT4pKRDfs/g0ZSqiobPBKtNMOvnBX+r9jW9vQhpQCeMTikZR5JIakaYWDLCJJMezM/PgMIj8z05nYNJaVR6GhU1KeyvSWVvTSp7D6WwuzqF3dXJlFelsKsqiZ0Hk9hxIJkde41llUlsrzT2HGw80aQlJ5EXJpi8ugmmgd5M5LGalOTmLcV/ImKaLMzsAuD/AcnAA+7+kzrL04FHgRFAGTDR3Veb2ReAnxD8PzgI/Ku7vxbLWEUkhlIzoaBP8GjKgb3BfU+qK4LeS1UFVFeGfw8E86sqg3lNzq8MllXuhuqt4fzKw9u2qgrS/RDp0MiJxg1IAs9KxVMyqEnJoCYpneqkdKosjYOWxgHSqCSNSk9lf00q+/ansG9PCnsOpR5OQls8hTUEyeoAqUF70jjgwXRyWiZp6VmkZmSRnplNZlYWWRlZ5GelHZVYunfI5Iy+hcfxDxO9mCULM0sG7gG+AKwH5prZC+7+cUSzW4Cd7t7PzCYBPwUmAtuBS9x9o5kNAl4BesQqVhFJIOk5waOlHKo6OolEnYgOYNUVWFUlSWHySTucoGq3t/dIwvNKoBJqKsEPRP/tezB8hCN5NdjRycVTWZ95CnzvT7HZP6FY9ixGAcvdfSWAmT0NXApEJotLgTvD6ZnAr83M3P2DiDaLgUwzS3f3iMI8IiLNIDk1eJzg/U6OSU1NUGfsOHpNSdWVZFVVkFFVQdaBCqoP7Kcgt1fMQ45lsugBrIt4vh4Y3VAbd682s3KgkKBnUesKYH59icLMpgJTAXr37t18kYuIxFJSEiRlBsNzx7sJID18tITEOXpSDzMbSDA09dX6lrv7dHcf6e4ji4qKWjY4EZF2JJbJYgMQ2TfqGc6rt42ZpRAcYyoLn/cEngNudPcVMYxTRESaEMtkMRc4ycxKzCwNmAS8UKfNC8BN4fSVwGvu7mbWAXgRmObub8UwRhERiULMkoW7VwNfJziTaQkww90Xm9kPzWxC2OxBoNDMlgPfBKaF878O9AN+YGYfho/OsYpVREQap/tZiIi0Y9HezyKhD3CLiEhiULIQEZEmKVmIiEiT2swxCzPbBqw5gU104uiLAROF4jo2iuvYKK5j0xbj6uPuTV6o1maSxYkys3nRHORpaYrr2CiuY6O4jk17jkvDUCIi0iQlCxERaZKSxRHT4x1AAxTXsVFcx0ZxHZt2G5eOWYiISJPUsxARkSYpWYiISJPaVbIwswvMbKmZLTezafUsTzezZ8Ll75pZcYLENdnMtkUUVby1heJ6yMy2mtmiBpabmf0qjHuBmQ1PkLjONrPyiP31gxaKq5eZvW5mH5vZYjP7l3ratPg+izKuFt9nZpZhZu+Z2UdhXP9ZT5sW/0xGGVdcPpPhayeb2Qdm9ud6lsVuf7l7u3gAycAKoBRIAz4CBtRp83+A34bTk4BnEiSuycCv47DPxgPDgUUNLL8IeBkwYAzwboLEdTbw5zjsr27A8HA6F/i0nn/LFt9nUcbV4vss3Ac54XQq8C4wpk6beHwmo4krLp/J8LW/CTxZ379XLPdXe+pZHL4nuLsfBGrvCR7pUuCRcHomcJ6ZWQLEFRfuPgvY0UiTS4FHPfAO0MHMuiVAXHHh7pvcfX44vYegNH+POs1afJ9FGVeLC/fB3vBpavioe8ZNi38mo4wrLsKbwn0JeKCBJjHbX+0pWdR3T/C6H5ij7gkO1N4TPN5xAVwRDlvMNLPY3509OtHGHg9nhMMIL4e3521RYfd/GMGv0khx3WeNxAVx2GfhkMqHwFbgb+7e4P5qwc9kNHFBfD6TdwPfAWoaWB6z/dWekkVr9ieg2N2HAH/jyC8Hqd98gno3pwH/C/yxJV/czHKAZ4FvuPvulnztxjQRV1z2mbsfcvehBLddHmVmg1ridZsSRVwt/pk0s4uBre7+fqxfqz7tKVmc0D3B4xmXu5e5+4Hw6QPAiBjHFK1o9mmLc/fdtcMI7v4SkGpmnVritc0sleAL+Ql3/0M9TeKyz5qKK577LHzNXcDrwAV1FsXjM9lkXHH6TI4FJpjZaoLh6nPN7PE6bWK2v9pTsjjue4LHO646Y9oTCMacE8ELwI3hGT5jgHJ33xTvoMysa+04rZmNIvh/HvMvmPA1HwSWuPtdDTRr8X0WTVzx2GdmVmRmHcLpTOALwCd1mrX4ZzKauOLxmXT377l7T3cvJvieeM3dr6/TLGb7K6U5NtIauHu1mdXeEzwZeMjDe4ID89z9BYIP1GMW3BN8B8E/SCLEdbsF9y2vDuOaHOu4AMzsKYKzZDqZ2XrgDoKDfbj7b4GXCM7uWQ7sB25OkLiuBG4zs2qgApjUAkkfgl9+NwALw/FugO8DvSNii8c+iyaueOyzbsAjZpZMkJxmuPuf4/2ZjDKuuHwm69NS+0vlPkREpEntaRhKRESOk5KFiIg0SclCRESapGQhIiJNUrIQEZEmKVmIJAALqr5+poqoSKJQshARkSYpWYgcAzO7PrzXwYdmdl9YcG6vmf0yvPfB382sKGw71MzeCYvNPWdmBeH8fmb2ali0b76Z9Q03nxMWpfvEzJ5ogYrHIlFTshCJkpn1ByYCY8Mic4eA64BsgitoBwJvElxRDvAo8N2w2NzCiPlPAPeERfvOBGrLfQwDvgEMILi/ydiYvymRKLWbch8izeA8goJxc8Mf/ZkEJaxrgGfCNo8DfzCzfKCDu78Zzn8E+L2Z5QI93P05AHevBAi39567rw+ffwgUA3Ni/7ZEmqZkIRI9Ax5x9+8dNdPsP+q0O94aOgcipg+hz6ckEA1DiUTv78CVZtYZwMw6mlkfgs/RlWGba4E57l4O7DSzceH8G4A3wzvVrTezy8JtpJtZVou+C5HjoF8uIlFy94/N7N+Bv5pZElAF/BOwj+AGOf9OMCw1MVzlJuC3YTJYyZEKszcA94XVQquAq1rwbYgcF1WdFTlBZrbX3XPiHYdILGkYSkREmqSehYiINEk9CxERaZKShYiINEnJQkREmqRkISIiTVKyEBGRJv3/KRK9UXr5AkwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt \n",
    "\n",
    "# let's double check that the model didn't overfit by comparing the training and validation loss\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model train vs validation loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6a. Predicting text normalization for test data\n",
    "\n",
    "I'll use my trained model to predict normalization for the full (10,000 samples) test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# load weights from training\n",
    "model = load_model('saved_model.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1088564, 3)\n",
      "(100000, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>before</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Another</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>religious</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>family</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>is</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>of</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>Hazrat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>Sayyed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>Ahmad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>his</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentence_id  token_id     before\n",
       "0            0         0    Another\n",
       "1            0         1  religious\n",
       "2            0         2     family\n",
       "3            0         3         is\n",
       "4            0         4         of\n",
       "5            0         5     Hazrat\n",
       "6            0         6     Sayyed\n",
       "7            0         7      Ahmad\n",
       "8            0         8        and\n",
       "9            0         9        his"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare test data in the right format\n",
    "X_test_data = pd.read_csv(\"en_test.csv\")\n",
    "X_test_data['before'] = X_test_data['before'].apply(str)\n",
    "print(X_test_data.shape)\n",
    "X_test_data = X_test_data[:100000]\n",
    "print(X_test_data.shape)\n",
    "X_test_data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = list(X_test_data['before'])\n",
    "X_test = [list(token) for token in X_test]\n",
    "\n",
    "X_test = index(X_test, input_vocab) # Convert to integer index\n",
    "X_test = add_context_window(X_test, context_size, padding_entity, input_vocab) # Add context window\n",
    "X_test = batch_wise_padding(X_test, X_seq_len) # Padding\n",
    "\n",
    "# convert X_test to integer array, batch-wise (converting full data to array at once takes a lot of time)\n",
    "# X_test = array_batchwise(X_test, X_seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000/100000 [==============================] - 709s 7ms/step\n"
     ]
    }
   ],
   "source": [
    "# make predictions\n",
    "test_predictions = np.argmax(model.predict(np.asarray(X_test), batch_size = 64, verbose=1), axis=2)\n",
    "\n",
    "predicted_test_sequences = []\n",
    "for prediction in test_predictions:\n",
    "    sequence = ' '.join([target_vocab_reversed[index] for index in prediction if index > 0])\n",
    "    predicted_test_sequences.append(sequence)\n",
    "np.savetxt('test_predictions', predicted_test_sequences, fmt='%s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6b. Investigating the model's predictions: what went wrong?\n",
    "\n",
    "I'll focus on the \"interesting\" cases - where the model didn't predict <self\\> or sil. These cass include dates, measures, money, and other cases that are not already trivially normalized. Per the concern I bring up in my write-up analysis that my model is only memorizing normalizations, I will also look at novel test cases that don't appear in my training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>before</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Another</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>religious</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>family</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>is</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>of</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>Hazrat</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>Sayyed</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>Ahmad</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>and</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>his</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentence_id  token_id     before   after\n",
       "0            0         0    Another  <self>\n",
       "1            0         1  religious  <self>\n",
       "2            0         2     family  <self>\n",
       "3            0         3         is  <self>\n",
       "4            0         4         of  <self>\n",
       "5            0         5     Hazrat  <self>\n",
       "6            0         6     Sayyed  <self>\n",
       "7            0         7      Ahmad  <self>\n",
       "8            0         8        and  <self>\n",
       "9            0         9        his  <self>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('test_predictions', 'r') as f:\n",
    "    predicted_test_sequences = f.readlines()\n",
    "    # need to get rid of \"\\n\" from loading\n",
    "    predicted_test_sequences = [pred.rstrip('\\n') for pred in predicted_test_sequences]\n",
    "    \n",
    "pred = pd.Series(predicted_test_sequences)\n",
    "\n",
    "X_test_data['after'] = pred.values\n",
    "\n",
    "X_test_data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>before</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>75</td>\n",
       "      <td>seventy five</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>76</td>\n",
       "      <td>seventy six</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>124</td>\n",
       "      <td>one hundred twenty four</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>pp</td>\n",
       "      <td>p p</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2010</td>\n",
       "      <td>twenty ten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>25 m</td>\n",
       "      <td>seventy meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>200</td>\n",
       "      <td>two hundred</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>two</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>3,400</td>\n",
       "      <td>thousand thousand five hundred</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>10,200 ft</td>\n",
       "      <td>hundred hundred million dollars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>7,000</td>\n",
       "      <td>two thousand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1895</td>\n",
       "      <td>eighteen ninety five</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>-</td>\n",
       "      <td>to</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>1945</td>\n",
       "      <td>nineteen forty seven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>W.</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>Pgs</td>\n",
       "      <td>p g</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>two</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>&amp;</td>\n",
       "      <td>and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>seven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>8</td>\n",
       "      <td>14</td>\n",
       "      <td>2010</td>\n",
       "      <td>twenty ten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>ten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>VPNs</td>\n",
       "      <td>d d &lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>14</td>\n",
       "      <td>9</td>\n",
       "      <td>22 February 2009</td>\n",
       "      <td>the twenty third of march two thousand nine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>16</td>\n",
       "      <td>14</td>\n",
       "      <td>8.5</td>\n",
       "      <td>point eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>3.9 kg</td>\n",
       "      <td>hundred point five meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>May 20, 2008</td>\n",
       "      <td>december twentieth two thousand eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>13,004</td>\n",
       "      <td>thousand two thousand three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>52.63 km2</td>\n",
       "      <td>point point per square kilometers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>2000</td>\n",
       "      <td>two thousand</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     sentence_id  token_id            before  \\\n",
       "30             2         4                75   \n",
       "35             2         9                76   \n",
       "40             2        14               124   \n",
       "43             2        17                pp   \n",
       "46             3         1              2010   \n",
       "52             3         7              25 m   \n",
       "56             3        11               200   \n",
       "65             4         1                 2   \n",
       "81             5         5             3,400   \n",
       "85             5         9         10,200 ft   \n",
       "92             5        16             7,000   \n",
       "100            6         5              1895   \n",
       "101            6         6                 -   \n",
       "102            6         7              1945   \n",
       "106            6        11                W.   \n",
       "113            7         0               Pgs   \n",
       "114            7         1                 1   \n",
       "116            7         3                 2   \n",
       "117            7         4                 &   \n",
       "118            7         5                 7   \n",
       "134            8        14              2010   \n",
       "154            9        13                10   \n",
       "213           13         9              VPNs   \n",
       "224           14         9  22 February 2009   \n",
       "252           16        14               8.5   \n",
       "255           16        17            3.9 kg   \n",
       "282           20         1      May 20, 2008   \n",
       "291           20        10            13,004   \n",
       "294           20        13         52.63 km2   \n",
       "300           21         1              2000   \n",
       "\n",
       "                                           after  \n",
       "30                                  seventy five  \n",
       "35                                   seventy six  \n",
       "40                       one hundred twenty four  \n",
       "43                                           p p  \n",
       "46                                    twenty ten  \n",
       "52                                seventy meters  \n",
       "56                                   two hundred  \n",
       "65                                           two  \n",
       "81                thousand thousand five hundred  \n",
       "85               hundred hundred million dollars  \n",
       "92                                  two thousand  \n",
       "100                         eighteen ninety five  \n",
       "101                                           to  \n",
       "102                         nineteen forty seven  \n",
       "106                                            w  \n",
       "113                                          p g  \n",
       "114                                          one  \n",
       "116                                          two  \n",
       "117                                          and  \n",
       "118                                        seven  \n",
       "134                                   twenty ten  \n",
       "154                                          ten  \n",
       "213                                   d d <self>  \n",
       "224  the twenty third of march two thousand nine  \n",
       "252                                  point eight  \n",
       "255                    hundred point five meters  \n",
       "282        december twentieth two thousand eight  \n",
       "291                  thousand two thousand three  \n",
       "294            point point per square kilometers  \n",
       "300                                 two thousand  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# boring_cases = ['<self>', 'sil.']\n",
    "# let's look at the interesting cases in the predicted for test data\n",
    "interesting_cases = X_test_data[X_test_data['after'] != '<self>']\n",
    "interesting_cases = interesting_cases[interesting_cases['after'] != 'sil.']\n",
    "interesting_cases.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 9863 novel cases in the test set.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>before</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>Haq</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>webapp</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>functionality</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Kuklick</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>Kuklick</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>Westcott</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>Kuklick</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>FINA</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>25 m</td>\n",
       "      <td>seventy meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>Dn</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>museum's</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>3,400</td>\n",
       "      <td>thousand thousand five hundred</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>10,200 ft</td>\n",
       "      <td>hundred hundred million dollars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>Spang</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>6</td>\n",
       "      <td>16</td>\n",
       "      <td>Wippich</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>Pgs</td>\n",
       "      <td>p g</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>8</td>\n",
       "      <td>15</td>\n",
       "      <td>ALA</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>8</td>\n",
       "      <td>18</td>\n",
       "      <td>Reluctant</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>Seinajoki</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>Ostrobothnia</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>Champ</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>Diagnosed</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>Arthritic</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>Jaw</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>outhouse</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>perched</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>VPNs</td>\n",
       "      <td>d d &lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>Soo</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>Birsa</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>Munda</td>\n",
       "      <td>&lt;self&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     sentence_id  token_id         before                            after\n",
       "13             0        13            Haq                           <self>\n",
       "17             1         2         webapp                           <self>\n",
       "18             1         3  functionality                           <self>\n",
       "26             2         0        Kuklick                           <self>\n",
       "31             2         5        Kuklick                           <self>\n",
       "36             2        10       Westcott                           <self>\n",
       "41             2        15        Kuklick                           <self>\n",
       "47             3         2           FINA                           <self>\n",
       "52             3         7           25 m                   seventy meters\n",
       "70             4         6             Dn                           <self>\n",
       "77             5         1       museum's                           <self>\n",
       "81             5         5          3,400   thousand thousand five hundred\n",
       "85             5         9      10,200 ft  hundred hundred million dollars\n",
       "107            6        12          Spang                           <self>\n",
       "111            6        16        Wippich                           <self>\n",
       "113            7         0            Pgs                              p g\n",
       "135            8        15            ALA                           <self>\n",
       "138            8        18      Reluctant                           <self>\n",
       "165           10         8      Seinajoki                           <self>\n",
       "170           10        13   Ostrobothnia                           <self>\n",
       "179           11         4          Champ                           <self>\n",
       "180           11         5      Diagnosed                           <self>\n",
       "182           11         7      Arthritic                           <self>\n",
       "183           11         8            Jaw                           <self>\n",
       "187           12         1       outhouse                           <self>\n",
       "196           12        10        perched                           <self>\n",
       "213           13         9           VPNs                       d d <self>\n",
       "218           14         3            Soo                           <self>\n",
       "226           15         0          Birsa                           <self>\n",
       "227           15         1          Munda                           <self>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now let's look at the novel test cases- those that didn't appear in the training data\n",
    "common = X_train_data.merge(X_test_data,on=['before'])\n",
    "novel_cases = X_test_data[(~X_test_data.before.isin(common.before))]\n",
    "print('There are {0} novel cases in the test set.'.format(novel_cases.shape[0]))\n",
    "novel_cases.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1778 novel important cases in the test set.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>before</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>25 m</td>\n",
       "      <td>seventy meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>3,400</td>\n",
       "      <td>thousand thousand five hundred</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>10,200 ft</td>\n",
       "      <td>hundred hundred million dollars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>Pgs</td>\n",
       "      <td>p g</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>VPNs</td>\n",
       "      <td>d d &lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>3.9 kg</td>\n",
       "      <td>hundred point five meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>13,004</td>\n",
       "      <td>thousand two thousand three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>52.63 km2</td>\n",
       "      <td>point point per square kilometers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "      <td>31 km</td>\n",
       "      <td>seventy meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>CUPS</td>\n",
       "      <td>s s u</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>31</td>\n",
       "      <td>9</td>\n",
       "      <td>11.8 MB</td>\n",
       "      <td>thousand hundred point point one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>33</td>\n",
       "      <td>10</td>\n",
       "      <td>534</td>\n",
       "      <td>three hundred thirty four</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>34</td>\n",
       "      <td>9</td>\n",
       "      <td>32 m</td>\n",
       "      <td>hundred seventy meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>E&amp;M</td>\n",
       "      <td>j d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>40</td>\n",
       "      <td>9</td>\n",
       "      <td>24 ft</td>\n",
       "      <td>seventy meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>601</th>\n",
       "      <td>42</td>\n",
       "      <td>6</td>\n",
       "      <td>3,500 kg</td>\n",
       "      <td>hundred hundred five meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>603</th>\n",
       "      <td>42</td>\n",
       "      <td>8</td>\n",
       "      <td>7,716 lb</td>\n",
       "      <td>three hundred hundred million dollars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>43</td>\n",
       "      <td>8</td>\n",
       "      <td>31 December 2014</td>\n",
       "      <td>the twenty second of march twenty fourteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>629</th>\n",
       "      <td>43</td>\n",
       "      <td>11</td>\n",
       "      <td>7,632</td>\n",
       "      <td>hundred six thousand thousand hundred thirty t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>704</th>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>http://www.britannica.com/EBchecked/topic/2852...</td>\n",
       "      <td>dot &lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>735</th>\n",
       "      <td>52</td>\n",
       "      <td>2</td>\n",
       "      <td>12 June 2007</td>\n",
       "      <td>the first of february two thousand seven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>742</th>\n",
       "      <td>52</td>\n",
       "      <td>9</td>\n",
       "      <td>http://search.eb.com/eb/article-9061377</td>\n",
       "      <td>one one one hundred fifty six</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>55</td>\n",
       "      <td>17</td>\n",
       "      <td>53 mi</td>\n",
       "      <td>hundred five meters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>826</th>\n",
       "      <td>57</td>\n",
       "      <td>6</td>\n",
       "      <td>UNESCO</td>\n",
       "      <td>s s &lt;self&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>832</th>\n",
       "      <td>58</td>\n",
       "      <td>2</td>\n",
       "      <td>J. K.</td>\n",
       "      <td>r j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>835</th>\n",
       "      <td>58</td>\n",
       "      <td>5</td>\n",
       "      <td>K.A.</td>\n",
       "      <td>m a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>71</td>\n",
       "      <td>5</td>\n",
       "      <td>A. M.</td>\n",
       "      <td>r d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1129</th>\n",
       "      <td>76</td>\n",
       "      <td>5</td>\n",
       "      <td>February 9, 1964</td>\n",
       "      <td>december third nineteen sixty five</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1137</th>\n",
       "      <td>76</td>\n",
       "      <td>13</td>\n",
       "      <td>OVP</td>\n",
       "      <td>t w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1148</th>\n",
       "      <td>77</td>\n",
       "      <td>9</td>\n",
       "      <td>24 km/h</td>\n",
       "      <td>hundred hundred per kilometers</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      sentence_id  token_id  \\\n",
       "52              3         7   \n",
       "81              5         5   \n",
       "85              5         9   \n",
       "113             7         0   \n",
       "213            13         9   \n",
       "255            16        17   \n",
       "291            20        10   \n",
       "294            20        13   \n",
       "356            25        12   \n",
       "418            30         0   \n",
       "434            31         9   \n",
       "467            33        10   \n",
       "478            34         9   \n",
       "503            36        11   \n",
       "573            40         9   \n",
       "601            42         6   \n",
       "603            42         8   \n",
       "626            43         8   \n",
       "629            43        11   \n",
       "704            49         1   \n",
       "735            52         2   \n",
       "742            52         9   \n",
       "800            55        17   \n",
       "826            57         6   \n",
       "832            58         2   \n",
       "835            58         5   \n",
       "1041           71         5   \n",
       "1129           76         5   \n",
       "1137           76        13   \n",
       "1148           77         9   \n",
       "\n",
       "                                                 before  \\\n",
       "52                                                 25 m   \n",
       "81                                                3,400   \n",
       "85                                            10,200 ft   \n",
       "113                                                 Pgs   \n",
       "213                                                VPNs   \n",
       "255                                              3.9 kg   \n",
       "291                                              13,004   \n",
       "294                                           52.63 km2   \n",
       "356                                               31 km   \n",
       "418                                                CUPS   \n",
       "434                                             11.8 MB   \n",
       "467                                                 534   \n",
       "478                                                32 m   \n",
       "503                                                 E&M   \n",
       "573                                               24 ft   \n",
       "601                                            3,500 kg   \n",
       "603                                            7,716 lb   \n",
       "626                                    31 December 2014   \n",
       "629                                               7,632   \n",
       "704   http://www.britannica.com/EBchecked/topic/2852...   \n",
       "735                                        12 June 2007   \n",
       "742             http://search.eb.com/eb/article-9061377   \n",
       "800                                               53 mi   \n",
       "826                                              UNESCO   \n",
       "832                                               J. K.   \n",
       "835                                                K.A.   \n",
       "1041                                              A. M.   \n",
       "1129                                   February 9, 1964   \n",
       "1137                                                OVP   \n",
       "1148                                            24 km/h   \n",
       "\n",
       "                                                  after  \n",
       "52                                       seventy meters  \n",
       "81                       thousand thousand five hundred  \n",
       "85                      hundred hundred million dollars  \n",
       "113                                                 p g  \n",
       "213                                          d d <self>  \n",
       "255                           hundred point five meters  \n",
       "291                         thousand two thousand three  \n",
       "294                   point point per square kilometers  \n",
       "356                                      seventy meters  \n",
       "418                                               s s u  \n",
       "434                    thousand hundred point point one  \n",
       "467                           three hundred thirty four  \n",
       "478                              hundred seventy meters  \n",
       "503                                                 j d  \n",
       "573                                      seventy meters  \n",
       "601                         hundred hundred five meters  \n",
       "603               three hundred hundred million dollars  \n",
       "626          the twenty second of march twenty fourteen  \n",
       "629   hundred six thousand thousand hundred thirty t...  \n",
       "704                                          dot <self>  \n",
       "735            the first of february two thousand seven  \n",
       "742                       one one one hundred fifty six  \n",
       "800                                 hundred five meters  \n",
       "826                                          s s <self>  \n",
       "832                                                 r j  \n",
       "835                                                 m a  \n",
       "1041                                                r d  \n",
       "1129                 december third nineteen sixty five  \n",
       "1137                                                t w  \n",
       "1148                     hundred hundred per kilometers  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# and now novel important cases\n",
    "# I want to see if my model generalizes or is just memorizing the training data\n",
    "novel_important_cases = novel_cases[novel_cases['after'] != '<self>']\n",
    "novel_important_cases = novel_important_cases[novel_important_cases['after'] != 'sil.']\n",
    "print('There are {0} novel important cases in the test set.'.format(novel_important_cases.shape[0]))\n",
    "novel_important_cases.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}