Fr2En_MachineTranslation_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.206336Z",
     "start_time": "2020-08-05T05:21:21.901608Z"
    },
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(2000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 2 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%autosave 2\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "from fastai.text import *\n",
    "from modules.callbacks import *\n",
    "from pathlib2 import Path\n",
    "import pickle\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from fastai.callbacks import lr_finder, SaveModelCallback, EarlyStoppingCallback,ReduceLROnPlateauCallback\n",
    "import math, copy, time\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "seaborn.set_context(context=\"talk\")\n",
    "from modules.model import *\n",
    "\n",
    "defaults.device = torch.device('cuda')\n",
    "#CUDA_VISIBLE_DEVICES=\"\"\n",
    "device =  torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Data (Freanch English Sentence Pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T17:09:32.220746Z",
     "start_time": "2020-08-04T17:09:32.199636Z"
    }
   },
   "outputs": [],
   "source": [
    "path = Config().data_path()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-08-04T17:10:41.790Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-08-04 22:40:41--  https://s3.amazonaws.com/fast-ai-nlp/giga-fren.tgz\n",
      "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.250.46\n",
      "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.250.46|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 2598183296 (2.4G) [application/x-tar]\n",
      "Saving to: ‘/home/santhosh/.fastai/data/giga-fren.tgz.1’\n",
      "\n",
      "giga-fren.tgz.1       2%[                    ]  59.94M  29.9KB/s    eta 7h 11m "
     ]
    }
   ],
   "source": [
    "! wget https://s3.amazonaws.com/fast-ai-nlp/giga-fren.tgz -P {path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! tar xf {path}/giga-fren.tgz -C {path} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Config().data_path()/'giga-fren'\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load DataSets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(path/'giga-fren.release2.fixed.fr') as f: fr = f.read().split('\\n')\n",
    "# with open(path/'giga-fren.release2.fixed.en') as f: en = f.read().split('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter Question pairs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use regex to pick out questions by finding the strings in the English dataset that start with \"Wh\" and end with a question mark."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_eq = re.compile('^(Wh[^?.!]+\\?)')\n",
    "re_fq = re.compile('^([^?.!]+\\?)')\n",
    "en_fname = path/'giga-fren.release2.fixed.en'\n",
    "fr_fname = path/'giga-fren.release2.fixed.fr'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = ((re_eq.search(eq), re_fq.search(fq)) \n",
    "        for eq, fq in zip(open(en_fname, encoding='utf-8'), open(fr_fname, encoding='utf-8')))\n",
    "qs = [(e.group(), f.group()) for e,f in lines if e and f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qs = [(q1,q2) for q1,q2 in qs]\n",
    "df = pd.DataFrame({'fr': [q[1] for q in qs], 'en': [q[0] for q in qs]}, columns = ['en', 'fr'])\n",
    "df.to_csv(path/'questions_easy.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.250818Z",
     "start_time": "2020-08-05T05:21:22.208172Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "## Writing processed data into pickle file\n",
    "\n",
    "#pickle.dump(qs, open('fr-2-en-data/fr-en-qs.pkl','wb'))\n",
    "qs = pickle.load(open('fr-2-en-data/fr-en-qs.pkl','rb'))\n",
    "\n",
    "en_qs,fr_qs = zip(*qs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.358553Z",
     "start_time": "2020-08-05T05:21:22.251954Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "path = Path()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.684955Z",
     "start_time": "2020-08-05T05:21:22.360184Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "en_tok = pickle.load((path/'fr-2-en-data/en_tok.pkl').open('rb'))\n",
    "fr_tok = pickle.load((path/'fr-2-en-data/fr_tok.pkl').open('rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.700848Z",
     "start_time": "2020-08-05T05:21:22.685957Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "TMP_PATH = path/'fr-2-en-data/tmp/'\n",
    "\n",
    "def load_ids(pre):\n",
    "    ids = np.load(TMP_PATH/f'{pre}_ids.npy',allow_pickle=True)\n",
    "    itos = pickle.load(open(TMP_PATH/f'{pre}_itos.pkl', 'rb'))\n",
    "    stoi = collections.defaultdict(lambda: 3, {v:k for k,v in enumerate(itos)})\n",
    "    return ids,itos,stoi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.978144Z",
     "start_time": "2020-08-05T05:21:22.701682Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "en_ids,en_itos,en_stoi = load_ids('en')\n",
    "fr_ids,fr_itos,fr_stoi = load_ids('fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fastai's Data Bunch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a function that collects batch of x,y pairs fills padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:22.999174Z",
     "start_time": "2020-08-05T05:21:22.979649Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "def seq2seq_collate(samples, pad_idx=1, pad_first=True, backwards=False):\n",
    "    \n",
    "    # unpack samples to tuples\n",
    "    samples = to_data(samples)\n",
    "    \n",
    "    # find max len of x, y batch wihich decides inp seq length\n",
    "    max_len_x,max_len_y = max([len(s[0]) for s in samples]),max([len(s[1]) for s in samples])\n",
    "    \n",
    "    max_len = max(max_len_x,max_len_y)\n",
    "    \n",
    "    # create a dummy tensor of height batch_size and width max_len with padded value\n",
    "    res_x = torch.zeros(len(samples), max_len).long() + pad_idx\n",
    "    res_y = torch.zeros(len(samples), max_len).long() + pad_idx\n",
    "    \n",
    "    # if backwards activate reverse mode used in bi-directional   \n",
    "    if backwards: pad_first = not pad_first\n",
    "    \n",
    "    # fill vocabulary with word indices\n",
    "    for i,s in enumerate(samples):\n",
    "        if pad_first: \n",
    "            res_x[i,-len(s[0]):],res_y[i,-len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])\n",
    "        else:         \n",
    "            res_x[i, :len(s[0])],res_y[i, :len(s[1])] = LongTensor(s[0]),LongTensor(s[1])\n",
    "    \n",
    "    # flip backward if backwards = True\n",
    "    if backwards: res_x,res_y = res_x.flip(1),res_y.flip(1)\n",
    "        \n",
    "    res_x_mask = (res_x != pad_idx).unsqueeze(-2)\n",
    "    res_y_mask = None\n",
    "    \n",
    "    # target mask creation\n",
    "    if res_y is not None:\n",
    "        dec_y = res_y[:, :-1]\n",
    "        tar_y = res_y[:, 1:]\n",
    "        decode_lengths = torch.tensor([len(s[1]) for s in samples])-1\n",
    "    return (res_x[:,1:],dec_y,decode_lengths), tar_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creates a Text databunch object with "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:23.110947Z",
     "start_time": "2020-08-05T05:21:23.000287Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "class Seq2SeqDataBunch(TextDataBunch):\n",
    "    \" decorator adds following method additionally to base class 'TextDataBunch\"\n",
    "    \n",
    "    ## Class method of Seq2SeqDataBunch (inherited from TextDataBunch) \n",
    "    @classmethod\n",
    "    def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=8, val_bs:int=None, pad_idx=1,\n",
    "               dl_tfms=None, pad_first=False, device:torch.device=device, no_check=True, backwards:bool=False, **dl_kwargs)-> DataBunch:\n",
    "        \n",
    "        \"Function takes pytorch dataset object transforms into 'databunch' for classification and cls will allow to access parent class methods just  like 'self'\"\n",
    "        \n",
    "        # store dataset obj into list\n",
    "        datasets = cls._init_ds(train_ds,valid_ds,test_ds)\n",
    "        val_bs = ifnone(val_bs, bs) #returns val_bs if none bs\n",
    "        \n",
    "        # stores raw seq of unequal len into tensor with padding\n",
    "        # below function takes batches output from Dataloader returns padded tensor\n",
    "        collate_fn = partial(seq2seq_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)\n",
    "        \n",
    "        #sampler function: generater takes dataset then sorts and returns sorted indices    \n",
    "        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs//2)\n",
    "        \n",
    "        # train data loader obj with Sortish sampler(sort with some randomness)\n",
    "        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)\n",
    "        dataloaders = [train_dl]\n",
    "        \n",
    "        # other dataloaders with pure sorting append into dataloaders list\n",
    "        for ds in datasets[1:]:\n",
    "            lengths = [len(t) for t in ds.x.items]\n",
    "            sampler = SortSampler(ds.x, key = lengths.__getitem__)\n",
    "            dataloaders.append(DataLoader(ds,batch_size=val_bs,\n",
    "                                          sampler = sampler,**dl_kwargs))\n",
    "        \n",
    "        return cls(*dataloaders, path = path, device = device, collate_fn=collate_fn, no_check = no_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:23.250304Z",
     "start_time": "2020-08-05T05:21:23.112353Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "class Seq2SeqTextList(TextList):\n",
    "    _bunch = Seq2SeqDataBunch\n",
    "    _label_cls = TextList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:23.439889Z",
     "start_time": "2020-08-05T05:21:23.251739Z"
    },
    "init_cell": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "pd.set_option('display.width', 10000)\n",
    "pd.set_option('display.expand_frame_repr', True)\n",
    "\n",
    "\n",
    "data = pd.DataFrame(qs, columns=['en', 'fr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-21T05:45:35.652983Z",
     "start_time": "2020-05-21T05:45:35.625698Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row0_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row0_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row1_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row1_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row2_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row2_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row3_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row3_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row4_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row4_col1 {\n",
       "            width:  300px;\n",
       "        }</style><table id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >fr</th>        <th class=\"col_heading level0 col1\" >en</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857level0_row0\" class=\"row_heading level0 row0\" >10557</th>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row0_col0\" class=\"data row0 col0\" >Quelle information devrait être fournie?</td>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row0_col1\" class=\"data row0 col1\" >What information should be provided?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857level0_row1\" class=\"row_heading level0 row1\" >5641</th>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row1_col0\" class=\"data row1 col0\" >Qu'est qu'un emploi?</td>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row1_col1\" class=\"data row1 col1\" >What is a job?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857level0_row2\" class=\"row_heading level0 row2\" >29027</th>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row2_col0\" class=\"data row2 col0\" >Qu'attendez-vous de la nouvelle structure ?</td>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row2_col1\" class=\"data row2 col1\" >What are your expectations from the newly established structure?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857level0_row3\" class=\"row_heading level0 row3\" >22777</th>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row3_col0\" class=\"data row3 col0\" >Quel est le meilleur résultat que vous puissiez obtenir de ce contrat?</td>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row3_col1\" class=\"data row3 col1\" >What is the best result that can be obtained from this agreement?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857level0_row4\" class=\"row_heading level0 row4\" >13895</th>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row4_col0\" class=\"data row4 col0\" >Quel contrôle peut-on conserver lorsque les données traverses les frontières?</td>\n",
       "                        <td id=\"T_14c8d186_cfe1_11ea_a8bb_2b7a2f75b857row4_col1\" class=\"data row4 col1\" >What control does one have when data crosses the border?</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8abe6dbed0>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sample(5)[['fr', 'en']].style.set_properties(subset=['fr', 'en'], **{'width': '300px'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-22T04:13:26.212097Z",
     "start_time": "2020-05-22T04:13:26.006457Z"
    }
   },
   "outputs": [],
   "source": [
    "!jupyter nbextension enable --py widgetsnbextension --sys-prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:23.570892Z",
     "start_time": "2020-08-05T05:21:23.441140Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "# lowercase\n",
    "data['en'] = data['en'].apply(lambda x:x.lower())\n",
    "data['fr'] = data['fr'].apply(lambda x:x.lower())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create a Seq2SeqTextList (Fastai's datablock api)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:30.988000Z",
     "start_time": "2020-08-05T05:21:23.572452Z"
    },
    "init_cell": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/santhosh/miniconda3/envs/CapGen/lib/python3.6/site-packages/fastai/core.py:302: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return np.array(a, dtype=dtype, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# text list with 'fr' column as label  and split list randomly (0.8) \n",
    "src = (Seq2SeqTextList.from_df(data, path = path, cols='fr')\n",
    "       .split_by_rand_pct(seed=42)\n",
    "       .label_from_df(cols='en',label_cls=TextList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:31.024751Z",
     "start_time": "2020-08-05T05:21:30.989100Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "src = src.filter_by_func(lambda x,y: len(x) > 30 or len(y) > 30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model initialize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:32.959916Z",
     "start_time": "2020-08-05T05:21:31.025670Z"
    },
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/santhosh/miniconda3/envs/CapGen/lib/python3.6/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n",
      "<string>:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"
     ]
    }
   ],
   "source": [
    "from modules.model import *  # BERT\n",
    "from modules.callbacks import *\n",
    "from transformer.model import Translation\n",
    "\n",
    "en_emb = torch.load('models/en_emb.pth')\n",
    "fr_emb = torch.load('models/fr_emb.pth')\n",
    "\n",
    "databunch = src.databunch()\n",
    "databunch.batch_size = 64\n",
    "\n",
    "\n",
    "#model = make_model(len(fr_stoi), len(en_stoi), N=6, h = 6, d_model=300,src_embedding = None,tar_embedding = None).to(device);\n",
    "model = Translation(src_vocab_size=len(fr_stoi),\n",
    "    tgt_vocab_size=len(en_stoi),\n",
    "    d_model=300, d_ff=2048,\n",
    "    d_k=64, d_v=64, n_heads=6, \n",
    "    n_layers=6, src_pad_index=1,\n",
    "    tgt_pad_index=1, device=device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-13T04:16:06.968618Z",
     "start_time": "2020-06-13T04:16:06.952009Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'src_vocab_size': 22188,\n",
       " 'tgt_vocab_size': 14236,\n",
       " 'd_model': 300,\n",
       " 'd_ff': 2048,\n",
       " 'd_k': 64,\n",
       " 'd_v': 64,\n",
       " 'n_heads': 6,\n",
       " 'n_layers': 6,\n",
       " 'src_pad_index': 1,\n",
       " 'tgt_pad_index': 1,\n",
       " 'device': device(type='cuda')}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(src_vocab_size=len(fr_stoi),\n",
    "    tgt_vocab_size=len(en_stoi),\n",
    "    d_model=300, d_ff=2048,\n",
    "    d_k=64, d_v=64, n_heads=6, \n",
    "    n_layers=6, src_pad_index=1,\n",
    "    tgt_pad_index=1, device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Fastai - with CrossEntropyLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-05T05:21:32.977952Z",
     "start_time": "2020-08-05T05:21:32.960772Z"
    },
    "hidden": true,
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "def CrossEntropy_loss(input,targets):\n",
    "    pred,_,_,_,decode_lengths = input\n",
    "    x,y = pred.contiguous().view(-1, pred.size(-1)), targets.contiguous().view(-1)\n",
    "    loss = nn.CrossEntropyLoss().to(device)(F.log_softmax(x, dim=-1), y)\n",
    "    return  loss #loss(pred.data.long(), targets.data.long())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T16:56:28.698814Z",
     "start_time": "2020-07-01T16:56:28.616475Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from modules.callbacks import *\n",
    "opt_fn = partial(optim.Adam, betas=(0.9, 0.98))\n",
    "\n",
    "#criterion = LabelSmoothing(len(en_itos), 1, 0.1).to(device)\n",
    "\n",
    "#learn = Learner(dataBunch,arch,loss_func= loss_func,opt_func=opt_fn, metrics=[topK_accuracy,BleuMetric(metadata,vocab)],callback_fns=[ShowGraph]) #,TeacherForcingTurnOff,TeacherForcingCallback\n",
    "\n",
    "\n",
    "# Fatai Learner object\n",
    "learn_CEL = Learner(databunch,model,loss_func=CrossEntropy_loss,wd=0.1, metrics=[seq2seq_acc, CorpusBLEU(len(en_itos))],callback_fns=[ShowGraph]) #,TeacherForcingTurnOff,TeacherForcingCallbacklearn = "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Learning rate finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T16:33:02.751747Z",
     "start_time": "2020-07-01T16:32:45.880112Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>seq2seq_acc</th>\n",
       "      <th>bleu_metric</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='89' class='' max='435', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      20.46% [89/435 00:16<01:02 20.2933]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 3.02E-05\n",
      "Min loss divided by 10: 1.20E-02\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEZCAYAAACJjGL9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5hU9dnG8e8z2ztllw7SQZqCooBYY+waE40FRbFiexM1mpi8xmiq0RijsVesQVFe0dgVOwIi0kQ6SC+7sLtsb7/3jzOL67oMW2b2zO7en+uaa90zZ848M4N7zzm/Zs45RERE9ibgdwEiIhLdFBQiIhKSgkJEREJSUIiISEgKChERCSnW7wLCycwq8MIv3+9aRERakHSgyjlXZyZYa+oea2ZVgGVkZPhdiohIi5GXlwfgnHN1XmVqVWcUQH5GRkZGbm6u33WIiLQY7dq1Iy8vb69XYtRGISIiISkoREQkJAWFiIiEpKAQEZGQFBQiIhKSgkJEREJqbd1jG+2D5dvZnl9C/05p9O+USkZSnN8liYhEBQVF0H/mrOedpdv2/N4pLYG+WSm0T44nJSGWlPgYUhNj6dYuiT6ZKfTJTKFLeiJVDnIKS9meX8qO3aUkx8fQJyuFrNQEzMzHVyQiEh4KiqCMpDg6pSWwfXcpANt3l+75771JiA1QUeWorPrh6PaUYGCM7t2BK4/qR6e0xIjULSISaa1tCo/cpo7MzisuZ/WOAlZtK2BtTiG7S8opKq2koLSC/JJyNuwsZnNeMXW9bfExAcoqq36wPTk+hssO78tlR/QlNUHZLCLRJTgyO885166u+xUUjVBSXsn6nUV8m1NEYlyAzumJdEpLICMpjpLyKtblFLI2u5BlW/J5Zva37CoqByAzNZ7Lj+jL2L6ZDO6aRlyM+hKIiP8UFD7LLynn4Y9W8/inaykp/+5sIyE2wLDuGRzQox2DuqQysHMaAzqn6YxDRJqdgiJKbM0r4d6ZK5n5zXa25pfsdb/eHZO55PC+nDO6p844RKRZKCii0Ja8Yhasz2XBhly+3pzPim27f9Bw3iczhRuPH8SJw7qo95SIRJSCooXILSpj+dbdvDx/Iy99uZHqjlQjemQwqld7stISyEpNoFN6AiN7ticjWeM8RCQ8FBQt0PKtu7nz7WW89832Ou8PGIzs1Z4jB2Zx1KAshnfP0FmHiDSagqIFm7t2JzMWbGJbfik7CkrJ3l3KtvwSKmqN2xjbtyP3nHMgndI1VkNEGk5B0coUl1Uye00OH63YwYfLt7MupwiAzNQE7j33QMb1y/S5QhFpaRQUrdw7X2/lhmkLyS+pIGBw3bEDufro/gQCuhQlIvWzr6BQ/8sW7rihXXj9F4czvHsGVQ7uencF5z02h3XZhX6XJiKthIKiFejZIZmXrhzLBWP3A+DzNTkc/6+PefDD1ZTXMaWIiEhD6NJTKzNz2TZu/r8lbM7zBvXt3zWdv58xnBE96jyjFBHRpae25pjBnXnn+iOZNK43ZvDNlnx+9sAsHv5oNVV1zHIrIrIvvgaFmR1mZjPNrNDM8s3sDTMb7mdNrUFqQiy3njaU6VeOY0CnVCqqHH97cxkXTfmC7ILQU6eLiNTmW1CY2RjgA7w1Mc4DJgGZwMdm1t+vulqTkb3a8+o14zn3kJ4AfLRiByfd8wmzVmX7XJmItCS+tVGY2TvAMKCfc644uK0dsAZ40zl3XiOO2ebbKPbm1YWb+d30xRSUet1obz55CBcd1lsjukUkqtsoxgIzq0MCwDmXC3wCnG5mMb5V1gqddkA3Xv/FeIZ1T6fKwR//u5TfTl9MWYV6RYlIaH4GRTxQ1wXzUiAZ6Fv7DjPLDXUDMiJcc4u2X8cUpk0ex8nDuwIw9YsNTHx8DjsLy3yuTESimZ9BsRQYYzWufZhZHHBI8FfNRREBSfEx3DdhJNceOwCAOWt38pP7P2Xltt0+VyYi0crPoPg3MAS418y6m1lP4BGgR/D+H1wTcc61C3UD8pqv/JbLzLj22IHcP2EUiXEBNuws5mcPzOLjFTv8Lk1EopBvQeGcewK4Ca+300ZgPTAI+Edwl83+VNZ2nDyiK9Mmj6NTWgK7Syu4aMoXPDP7W7/LEpEo4+s4Cufc3/EuMQ0HejvnxgEdgG+dcxv8rK2tGN4jgxnXHMbQbulUVjl+/8oS/jBjCYWlFX6XJiJRIqqm8DCz3sBC4E/OuX+E3rvOx6t7bCMVllZw7QsLeHfpNgAykuK4cOx+XDiuNx1TE3yuTkQiKWqnGTezA4DTgXl4PZ0OBH4LzAdOdM41+CutgqJpqqoc/3p/JQ9/tJrSYLfZxLgAZx/ck+uPG0RGkpZfFWmNojkoBuM1Xg8DUvAG2j0N/NM516h5JhQU4ZFTUMpTs9bx1OffkldcDsDAzqlMuegQurVL8rk6EQm3qA2KSFBQhFdhaQXPzv6Wf7yznPJKR+f0BKZcdAj7d033uzQRCaNoHpktUS4lIZbJR/bjqYsOIS0hlm35pfz8oc/5dKXmihJpSxQUsk/j+mcy7cqxdM1IpKC0gklPzuWDZdv9LktEmomCQuplcJd0pl81jsFd0qioctz8yhKKyyr9LktEmoGCQuqta0YSD088iPiYAJtyi3nk4zV+lyQizUBBIQ2yX8cULjm8DwAPfrSKzbnF+3iEiLR0CgppsKuP7k9WWgIl5VX8/a1lfpcjIhGmoJAGS02I5dfHDwJgxoLNzFu30+eKRCSSFBTSKGeM6sGIHt7yH7e9tpSqqtYzHkdEvk9BIY0SCBh/OHUoAIs35THtS83hKNJaKSik0Q7arz2nH9gNgD//9xs27CzyuSIRiQQFhTTJzacMITPVW8/i+hcXUKlLUCKtjoJCmiQzNYF//HwEAF+s28WDH67yuSIRCTcFhTTZUYM6MWlcbwDufm8lCzZoUkaR1kRBIWFx04mDGdQ5jcoqxy+nfqUV8kRaEQWFhEViXAz3nHsg8bEBvs0p4i9vfON3SSISJgoKCZvBXdL3DMSbOnc9a7MLfa5IRMJBQSFhdeG43vTskESVg3vfX+l3OSISBgoKCau4mAC/OGYAADMWbGLV9t0+VyQiTaWgkLD76cju9O6YTI+dW9h83sWQng6BgPfzqqtg9Wq/SxSRBtCa2RIRs+57hgOvu4zYqgriq2oscBQX591eeglOPNG/AkVkj32tma2gkPBbvRo3YgRWFGJKj+RkWLQI+vVrvrpEpE77CgpdepLwu+surLw89D7l5XD33c1Tj4g0ic4oJPzS02F3PRqx09MhLy/y9YhISDqjkOZXUBDe/UTEVwoKCb/U1PDuJyK+UlBI+J1/vtezKZS4OJg4sXnqEZEmUVBI+P3qV/ULiuuua556RKRJFBQSfv36eeMkkpN/EBhlgRjKE5K8+9U1VqRFUFBIZJx4ojdO4vLL94zMLk1O5T8HnMBJl9zH1nFH+12hiNSTusdKsykoreCoOz8ku6CUk4Z34e6zDyQhNsbvskTaPHWPlaiRmhDLDccNBOCNxVs57u6PeWvJFlrTlxWR1khBIc3qrIN7MvmIvsQGjG9zirji2fmc88hslmzSwDuRaKVLT+KL1TsK+Ovr3/D+su0AxAaMm0/enwvH9cbMfK5OpG3RpIAS1T5dmc2tr33Nqu3eKO0zRvXgLz8dRmKc2i5EmovaKCSqjR+QyavXHMbJI7oC8PL8jZz18Odszi32uTIRqaagEN8lx8dy37kjuenEwQQMFm3M47T7PmPDzhDTlItIs1FQSFQwM644sh9TLjqE9MRYsgtK+dW0hVRWtZ5LoyItlYJCosoRA7O455yRAMxdu5PHP13jc0UioqCQqHP04E6cd2gvAP7x9gq+2ZLvc0UibZuCQqLS/568P707JlNWWcV1LyygtKJy3w8SkYhQUEhUSo6P5Z9nH0jAYNnW3fzznRV+lyTSZikoJGqN6tWea47uD8Ajn6xh7tqdPlck0jYpKCSq/c+PBjC8ewbOwW+nL9IlKBEfKCgkqsXFBLj9jOHEBIzVOwp56EP1ghJpbr4GhZmNNLNXzGyzmRWa2VIzu8nMEvysS6LL0G4ZXDq+DwD3f7Bqz3QfItI8fAsKMxsMzAJ6A9cCpwLTgb8Aj/pVl0SnXx47gJ4dkiirrOJ3/7eYKg3EE2k2fp5RnAMkAmc45150zs10zt0MPA+ca2b7WHRZ2pLk+Fj+fPpwwBuIN+3LDT5XJNJ2+BkU5cGftRciyAvep1ZL+Z4jB2bxkwO7AfDXN5axPb/E54pE2gY/g+IZYCfwoJn1MbN0M/sJcCFwl3OuysfaJEr9/pQhZCTFkVdczoTH5rA1T2EhEmm+BYVzbj0wBhgCrME7k3gFuNc59/u6HmNmuaFuQEazvQDxRWZqAneffQDxMQFWbS/gzIdmsS670O+yRFo1Pxuz9wNeA3YAPwWOBm4FfmVmf/KrLol+xwzuzJMXjSY5PoaNu4o586HPNR+USAT5tsKdmf0HOALo75wrrrH9D8AtQD/n3LoGHlMr3LUh89fv4qInvyCvuJz0xFieuvgQRvZq73dZIi1ONK9wNxJYWjMkgubh1TW4+UuSlmRUr/a8MHkMWWkJ5JdUcMHjc1m0UV8SRMLNz6DYDAwzs+Ra28cGf25q5nqkBRrcJZ1pk8fSOT2B3aUVnP/YHJZsqt2RTkSaws+guBfoArxtZmeY2bFm9mfg18B7zrnFPtYmLUjvzBSev+y7M4vzH5+jNguRMGpwUJhZfzM7oda2Q83sNTP7zMwur89xnHOvAD8GSoEHgBl4jdp/Ak5vaF3StvXLSuX5Sw+lY0o8uUXlnPfYHFZs2+13WSKtQoMbs83sZaCDc+7o4O+ZwAogFSgO/jwjGATNSo3Zsnzrbs555HN2FZWTmZrAC5PH0C8r1e+yRKJaJBqzDwbeq/H7uUA6MArIAuYAv2zEcUWabFCXNJ67dAwZSXFkF5Qy4dHZfJujcRYiTdGYoMjCa4iudgLwmXNuiXOuDJiKN4hOxBdDuqXzzCWHkJYQy7b8UiY8OoeNu4r8LkukxWpMUBQC7QDMLAYYD3xc4/5ivDMMEd+M6NGOKRcfQnJ8DJtyi5nwqKb7EGmsxgTF18BEM+sIXIbXJvFujfv3wxttLeKrg/ZrzxOTRpMYF2D9ziIunvIFZRWaQkykoRoTFHcCI4DtwP3AV8AnNe4/Dpjf9NJEmm5M3448MvFgzGDplnzufX+l3yWJtDgNDgrn3OvAMcC/gNuA41yw61TwLGMjMCWMNYo0yREDs5h8RD8AHvhwFfPX7/K5IpGWxbe5niJB3WNlb0orKvnJfZ+xbOtu+mSm8MYvDicpPsbvskSiQrPM9WRmscHR1ZeZWZdwHFMknBJiY7jrrAOIizHWZhdy+5vf+F2SSIvRmJHZd5jZFzV+N7xxFS8CDwOLzaxf+EoUCY+h3TK49tiBADz1+bd8ujLb54pEWobGnFGcwPcbr0/Fmy78TmBCcNtNTaxLJCImH9GXUb28s+tfv7SQorIKnysSiX6NCYqeQM2uI6cCa51zNznnpgIPAT8KR3Ei4RYbE+Cusw4kPjbA5rwS7n1/ld8liUS9xgRFPFBZ4/ej+f6UHmuArk0pSiSS+mSmcNVR3tXRxz5Zw0pNHigSUmOCYgPeWteY2VCgL/BRjfs7AQVNL00kcq44sh/7dUymosrx+xlLaE29/0TCrTFBMRW40Mz+C/wXyAfeqHH/SGB1GGoTiZjEuBhuO20oALPX7GTGgs37eIRI29WYoPgb3oC6sYADLnDO5QKYWQZwGvB+uAoUiZSjBnXixGFeb+4/v/4NecXlPlckEp0aMzK71Dl3iXOuo3Our3Pu1Rp378Zrn7g1XAWKRNLvTxlCcnwM2QWl/POd5X6XIxKVwroUqnOuyjmX55zTVzNpEbq1S+LaYwcA8PRsja0QqUujgsLMUszsNjNbZGYFwdsiM7vVzFLCXaRIJF10WB8O6NkO5+DaF75ie76mIxepqTEjszsAc4HfA13wZo/9CugM3ALMDe4j0iLExQS479yRpCfGkl1Qxi+mfkVllXpBiVRrzBnFH4HBwDVAV+fc4c65w4FuwNXAINRGIS1Mzw7J/OPnBwBeL6h7NB25yB6NCYrTgMeccw845/YMvHPOVTrnHgSeAE4PV4EizeW4oV24ZHwfAP49c6XaK0SCGhMUnfEuNe3N/OA+Ii3Ob04YvKe94pdTv2JddqHfJYn4rjFBsQ1vUN3ejAzuI9LixMd67RUZSXHkFJZx3mNz2JRb7HdZIr5qTFC8BlxiZpPNbM/jzSxgZpcDFwOv7vXRIlGuZ4dknrxoNCnxMWzKLea8R2erJ5S0aQ1e4S643OnnQD9gB1A9SmkQkAWsAsY553LCWGd9a9MKdxI2s9fkcOETcymtqGJg51SmXj6WDinxfpclEnZhX+EuGAAHA7cDOcDo4C0bb3qPg/0ICZFwG9O3Iw9PPIi4GGPFtgIueGIOhaVav0LankYNuHPO5Tvn/tc5N9Q5lxy8DXPO3QxMMLOlYa5TxBdHDerEv88dRUzAWLIpnz++pn/a0vaEdQqPoEy8y1AircIJw7rw2xMHA/DCvA28tWSrzxWJNK9IBIVIq3PxYX0Y3z8TgJumL2KbGrelDVFQiNRDIGDcddYBtEuOI7eonBumLaRK03xIG6GgEKmnzumJ3P6z4QB8sjKbJz5b63NFIs1DQSHSACcM68rZB/cE4I63lrNwg7piS+sXW5+dzOz6BhzzsEbWItIi3HLqEOaszWFdThEXT/mCl64cR59Mza4vrVe9BtyZWVUDj+ucczGNK6nxNOBOmsuq7QWc+dAscovK6dkhiZevHEentES/yxJplH0NuKtvUBzZ0Cd2zn3U0Mc0lYJCmtNX63cx4dE5FJdXsn/XdF6YPIb0xDi/yxJpsLAERUuhoJDm9sHy7Vz61Dwqqxxj+nbgqYsPISG22U+mRZok7FN4iMh3jh7UiTvOGAF4Cx7dOG2Rus1Ksyspr2TavA1s3x2Z8T0KCpEmOuOgHvzmBG/k9qsLN3P3eyt8rkjamtlrcrjxpUWM/dtM8kvKw358BYVIGFxxZF/OPaQXAP+euYpp8zb4XJG0JR8u3wHAAT0yItJOpqAQCQMz448/GcrhA7xpPn47fTGzVmkpVYk85xwzl20HvEuhkaCgEAmTuJgAD5w3isFd0qiockx+9ktWbd/td1nSyq3NLmT9ziIAjh6soBCJemmJcTw+aTRZaQnsLqng6ue+oqS80u+ypBX7IHjZKSstgSFd0yPyHAoKkTDr3i6JhyceREzAWL5tN7e/uczvkqQV+3C5d9npyIFZBAIWkedQUIhEwKhe7bnu2AEATJm1jpnLtvlckbRGRWUVzFmzE4hc+wQoKEQi5sqj+nNInw4A3DhtUcT6uEvbNWtVDmWVVcQEjPHBjhSR4GtQmNkUM3Mhbl38rE+kKWICxt1nH0h6Yiw5hWXcoMF4EmYfBC87HbRfezKSIjd9jN9nFH8Cxta6HQEUA3Occ1pzUlq07u2S+GtwDYuPV+zg3pkraU3T5oh/nHN7xk9E8rIT+BwUzrnVzrnZNW9AFpAEPO5nbSLhcsqIbvz8oB4A/Ou9ldz8yhIqKhs6IbPI963cXsCm3GIAjh6cFdHn8vuMoi4XA0XAC34XIhIufzp9GKeM6ArAc3PWc+nT8ygorfC5KmnJPggOsuuakcigzmkRfa6oCgoz6wqcALzknMv3ux6RcEmMi+Hec0Zy9dH9AG/KhTMfnMWWvGKfK5OWqrp94qhBWZhFpltstagKCuBCIIa9XHYys9xQNyCjWasVaYBAwLjx+MH8/YzhxAaMZVt3c+ETcymt0IA8aZjdJeXMW7cLgKMi3D4B0RcUk4BVzrmP/S5EJFLOHt2LJyaNJiZgrNhWwP0frPa7JGlhPluVTUWVIy7GOKx/5LrFVouaoDCz8cAg4Mm97eOcaxfqBuQ1W8EiTXDEwCwuO7wvAA98sIpvtuhKq9Tfl996ZxMje7YnNSE24s8XNUGB14hdCTzldyEizeHaYwfQJzOFiirHb15epJ5QUm9Lg18shnVvnqvtUREUZpYC/Bx42zm3ye96RJpDYlwMtwfHWCzamMeTn63ztyBpEZxzLN3sBcWQbpGZBLC2qAgK4GwgFXjC70JEmtOhfTty/hhvwaO73l3OuuxCnyuSaLctv5RdRd4qdpGaLba2aAmKi4Bs4FW/CxFpbr85YTDdMhIpKa/i1y/pEpSEtnSL1xQbF2P075TaLM8ZFUHhnDvcOZflnAv/Yq8iUS4tMY6/BC9BzV23kz+//o3PFUk0q77sNKBTGvGxzfMnPCqCQqStO3pQJ646yhuMN2XWOl74Yr3PFUm0qm7Ibq72CVBQiESNG44bxLH7e4Onbn5lCfPW7fS5IolGexqym6l9AhQUIlEjEJyWfECnVMorHVc8++WeSd9EAApKK1iX462PrTMKkTYqLTGORy84mIykOLILyrj86XkUlWnyQPEsqzEwc3+dUYi0Xb0zU7h/wihiAsbXm/O5/oWFWvBIgO/aJ3q0T4roQkW1KShEotD4AZnccsoQAN76eit3vbvc54okGvjRPgEKCpGodeG43lwwdj8A7v9gNdPnb/S5IvFb9RlFc152AgWFSFS75ZQhHD7Amx30ppcXqydUG1ZRWcWyrbuB5m3IBgWFSFSLjQlw34RR9MtKoayyisuensenK7P9Lkt8sCa7kLIKb9S+Lj2JyPdkJMXxxKTRdEiJZ1dROROfmMM/31lOpRq425Tq9om0xFh6tE9q1udWUIi0APt1TGHG1YdxQI8MnIN7Z65iwqOz2ZZf4ndp0kz2jMjumh7xpU9rU1CItBA9OyQz7YpxXHxYHwDmrN3JSfd8wqKNuT5XJs3hGx+m7qimoBBpQeJjA9xy6hAenngQ6Ymx5BSWMeHROXyhRu5W7XtrUDRz+wQoKERapOOHdmH6VYfRJT2RgtIKJj4+h09W7vC7LImQ7btLySksA3RGISIN0L9TKtOuGEvPDkmUlFdxyZR5vP31Vr/LkgioPpuIizEGdEpr9udXUIi0YD07JDNt8rg93Wevem4+byze4ndZEmbVDdn9m3ENipoUFCItXJeMRF6cPJYhXdOprHL84j9f8d7SbX6XJWG0cIPXYcGP9glQUIi0Ch1TE3ju0kMZ3CWNiirHVc/N56MVarNoDZxzLAgGxYG92vlSg4JCpJVonxLPs5ceuucy1OVPz+Pz1Tl+lyVNtCWvhO27SwEY2VNBISJNlJmawPOXjaF3x2RKK6q45Kkv9nwblZap+vNLiA0wqEvzN2SDgkKk1emcnsjzl42hR/skisoqufq5+ewKdq2Ulqc6KIZ3zyAuxp8/2QoKkVaoW7sknpw0mqS4GDblFnP9iwu0+FELtWB9sH3Cp8tOoKAQabUGdE7jrz8bBsAHy3fw8MdrfK5IGqqisorFm/IA/xqyQUEh0qr9dGQPzhndE4B/vLOcuWs11UdLsnzbborLKwGdUYhIBN162lAGd0mjssrxP/+ZT3ZBqd8lST1Vt09kpibQvV3zTi1ek4JCpJVLjIvhgfNGkRIfw7b8Uq56bj6lFZV+lyX1ULN9ormnFq9JQSHSBvTNSuXOnx8AwNy1O/nd9CU4p8btaLcwOIX8SB/bJ0BBIdJmnDS8KzccNxCAl+dv5IEPV/tckYSyu6ScldsLAH/bJ0BBIdKmXH10f342qjsAd769XBMIRrHFG/NwDsxgRI8MX2tRUIi0IWbG3342nEN6dwDguhcWaOR2lPoq+Ln0z0olLTHO11oUFCJtTEJsDA9NPIj9gtN8XPrUF2zYWeR3WVLLnokAfb7sBAoKkTapQ0o8T0waTUZSHNkFZUx6ci65RZrmI1pEw4yxNSkoRNqoflmpPHrBwcTHBFi9o5DLn/lS3WajxOa8EnYEZ4zVGYWI+OqQPh2466zvus3eOG2R5oSKAtXjJ5LiYhjU2Z8ZY2tSUIi0cace0I2bThwMwKsLN3PXu8t9rqhtc84xc9l2wJsxNtanGWNr8r8CEfHd5CP6ct6hvQC4/4PVvKlus77IKy7nime/5OX5GwE4rH+mzxV5FBQigplx22lDGdu3IwA3TFvIqu27fa6qbVm4IZeT7/2Et7/21jv/6cjuTD6yr89VeRQUIgJAbEyAf08YSdeMRArLKpn8zJfsLin3u6w2Yerc9Zz50Cw27iomITbAHWeM4J9nHUBiXIzfpQEKChGpITM1gQfOG7WnJ9SN0xZpTqgIe/Kztdw0fTHllY5+WSnMuOYwzhrd09dJAGtTUIjI94zs1Z5bTxsKwFtfb+XBjzQnVKQ89skabnttKQCH9e/Iq9eMZ3CXdJ+r+iEFhYj8wLmH9OSsg3sAcMdby5mxYJPPFbU+D3+0mj+//g0Ahw/I5PELR5OSEOtzVXVTUIjID5gZf/zJMEb3bg94jdsfr9jhc1WtxyMfr+Zvby4D4KhBWTx6wcFR0x5RFwWFiNQpMS6Gxy4YzaDOaZRXOq549ksWagLBJluyKY/bgyHxo8GdeHjiQVEdEqCgEJEQMpLjePqSQ+jeLomiskoumvIFq3cU1Pvx7y7dxg3TFvLM5+tYuW13m28Yr6is4rfTF1PlYECnVB44fxQJsdEdEgDWmj44M8vNyMjIyM3Vtx6RcFqzo4AzH/qcnYVldG+XxItXjN3nGs4fLNvOpU/Po7LGlCAdU+I5tG8HDuufyfj+mfTqkBxVvXsi7bFP1uxpl3jpirEcHJzu3W/t2rUjLy8vzzlX58RSvgeFmR0F/A44BIgH1gH/cs490ohjKShEImThhlzOfXQ2RWWV9MlM4YXJY+iUlljnvl+t38WER+dQXF5J93ZJlFVW7ZnkrqYe7ZMY27cj7ZLjCASMGDNiYwLs1yGZod3T6Z+VGhVTWITDxl1F/PifH1NcXsl5h/biLz8d7ndJe0R1UJjZhcDjwKPADKAcGBys675GHE9BIRJBn6/OYdKTcymtqGJg51SmXj6WDinx39tn9Y4CznxwFruKyunZIYmXrxxHVmoCa7ILmb0mh1mrcpi1OptdRfsezBcfG2BwlzTG9O3I8UM7M7Jne13OBPIAAA9oSURBVAKBlncG4pzjkqfmMXPZdrLSEnjv+iPJSPJ3MaKaojYozKwnsBy41Tl3R5iOqaAQibAPl2/nsqfnUV7pGNotnecvG0NGUhzOOTbuKuacR2azKbeYjinxvHTlOPpkpvzgGFVVjqVb8vl0VTYL1udSWlFJpfO2l1ZUsmJbAXnFPwySTmkJ/HhIZ44Z3InRfTqQ7vPKb/X130Wbueb5rwB44LxRnDS8q88VfV80B8WtwK+Bjs654jAdU0Eh0gze/norVz03n8oqR2ZqPGDsKirb0x6RHB/D1MvHMKJH49ZScM6xOa+ErzflsWhjHu99s41lW78/91TAvNlVx/TryHFDOnPQftFxvb+2DTuLOP3+z8gpLOPY/Tvx6AUHR127TDQHxUwgA/gncAvQH9gCPAvc4pz7wXJbZravBMjIyMhAQSESeTMWbOLaFxZQ+09ISnwMD5x/EEcOzArr863LLuSdpVt5++ttLNiQ+71GcvDGI/z6+MEM6RY9I5t3FZZxxoOzWJNdSEZSHG/88vB9dgLwQzQHxTKgG1AB/B74GjgG+C3wonPuvDoeo6AQiSKLN+Yx79uddEiJp31yPB1S4unVMTnil4QKSiuYt24nn6/J4aPlO/acbZjBTw7oxjXH9KdvZqqv7Rkl5ZWc99gcvvx2F/GxAZ679FBGR0kvp9qiOShWAAOAc51zU2tsvxO4ARjgnFvVwGPq0pNIG+Oc492l27jz7eWs3P7dGI+E2AB9MlPok5nCgE6pHNirHSN7tqd9rcb32vKKy/lk5Q7W7yxi465iNu0qJqewlHH9Mpl8RF86pibss6bKKsdVz33J219vwwzunxB97RI1RXNQfA6MATo453bV2H4M8D5wtnPuxQYeU0Eh0kZVVjlenr+Rf727gs15JXvdr29mCiN7tefAXu04sEc7BndNI8aM2WtyeHHeBt5cspXSiqo6H5sSH8PF4/tw6eF96+y1tKuwjEWb8pg+fyMzFmwG4JZThnDx+D7heZEREs1B8QhwGT8Mih8B7wE/d8691MBjKihE2riKyiq+3VnEmh2FrM0uYM2OQpZuyWfp5nwq6lgPPCE2QEZSHNtrjPNIjAswoFMa3dsl0aN9ErExAf4zd/2enljpibGM7NWe6jZp52BtdiHrdxZ979iXju/DzacMidyLDZNoDooTgDeB851zz9XYfhdwHdDHOfdtA4+poBCROhWXVbJ4Ux7z1+/iq/W7WLghj6353z/zOGi/9px9cE9OGtGV1FozueYVl/P4J2t4/NO1FJZV7vV5AgYDO6dx4rCu/M8x/VvEuI+oDQoAM3sDGAf8ge8as38NPOKcu6oRx1NQiEi9bc0rYcGGXDbnFnPEwEz6d0rb52NyCkqZ9uVGdhV+1zHTAZ3TEzmgRwZDu2WQFB/98zfVFO1BkQLcBpwLZAHrgceAO5xzdV8kDH08BYWISAPtKyh8XSXDOVeI18PpBj/rEBGRvWsds22JiEjEKChERCQkBYWIiISkoBARkZAUFCIiEpLvK9yFk5lVAZaRkeF3KSIiLUZeXh6Ac87VefLQ2oKiAu8sKd/vWlqQ6lTN87UKqYs+m+jV2j6bdKDKOVfnkIlWFRTScNVTt+9toI34R59N9Gprn43aKEREJCQFhYiIhKSgEBGRkBQUIiISkoJCRERCUlCIiEhICgoREQlJ4yhERCQknVGIiEhICgoREQlJQSEiIiEpKFoQM+thZveY2admVmBmzsyOitBzTTCzhWZWYmYbzex2M0ustc+kYA113RL3duzWKAo/m9PM7AMz22pmpWa2xcxeM7MxkagpmkXhZzPUzB40s7nB/ZyZ9Y5EPeGioGhZ+gPnAgXA+5F6EjM7H3gO+Aw4EfgrcDUwZS8PuQAYW+tWGqn6olS0fTaZwBfANcBxwHVAFvCJmY2PVH1RKto+m4OBU4GtwX2jn3NOtxZyAwI1/vt0wAFHhfk5YoAtwIxa2y8LPt+hNbZNCm470O/3xu9btH02e3l8Ol6AP+H3+9WWP5ta9VwbvL+33+9TqJvOKFoQ51xVffYzs0Qz+6OZraxx2eE+M0urx8PHAF2Ap2ptfw4oB85oWNVtQwv5bArwgqK8PrW2FtH22dS3nmhS5yIV0nKZWQzwOnAQ8DdgHjAY+BMw3MyO3sc/1GHBn0tqbnTOFZnZ6hr31/SWmWUBucC7wM3OuVVNeyWtjx+fTfA5A0B34LeAAQ808aW0Oj79f9NiKChan7OBY4CTnXNvBLe9b2YbgVfwrp2+HuLxHYM/d9Zx384a94N3jfUvwGxgNzAK74/RHDMb7Zxb0+hX0To152dTbQ7eHz/wLo2c4Jxb2NDC2wA/PpsWQ5eeWp+TgBzgHTOLrb7hfdOvBI4E7xtUzfvNrPa/hb0N2d+z3Tn3lnPuZufcf51zHznn7sb7ny0V+F24X1gr0GyfTQ0TgUPxLn0sAd6MVI+fFs6Pz6bFUFC0Pp3xvr2U17oV4jW4ZQb3e7/W/U8Et+cEf9b1DagDdX9j2sM5twT4Eq/nk3xfs382zrlvnHNznXPT8b4VrwHuCceLaWV8/f8m2unSU+uTDWwDTglxP8BkIK2O7V8Hfw4DVlTfaWbJQD/gtXrUEABaXINdM/D1s3HOVZrZl8CEhpXdJkTD/zdRS0HR+ryJd721yjk3f287OeeW7+Wu2XhtDxOB6TW2nwvE1dr2A2Y2DBgJPN2AmtsKvz+bBGAcoI4GP+TrZxPtFBQtjJmdGfzP0cGfR5pZJlDonHsTeB5vANxbZnY33mUgB/QEjgfucc7N2tvxnXMVZnYTMMXM7gNeAvYH/g685JybXaOWd4GZeN+mCvAC4jdAHl4jd5sShZ/NR3jtErnAfsAVwEDgZ2F6yS1GlH02yXhtIgAHBH+eaGY7gB3OuY+a/orDzO+BHLo17Ib3j7eu27oa+8QDNwGLgRK8P9yLgX8BXev5POcHH1MKbALuAJJq7XM3Xkjk412vXQ88BvTy+33SZ8NtwHxgV/Cz2Qr8HzDe7/dJnw29Q9Tzod/vVV03rUchIiIhqdeTiIiEpKAQEZGQFBQiIhKSgkJEREJSUIiISEgKChERCUlBIdIEZtY7uJTlrX7XIhIpCgppdmZ2VPCP6w1+19JamNmt9v01y6vMbKeZvW9mp4Xp+KeHo1ZpeTSFh0jTfAskARV+FxJ0C7AW7//tfniT2M0ws/Odc8814bh/wFu97ZWmlygtjYJCJMjM0pxzuxvyGOdNbVASoZIa403n3LzqX8zsJWAB3tQUTQkKacN06UmimpklmNnvzOxrMysxs1wze83MRtbaL2Bm/2tmH5vZVjMrM7P1ZvagmXWste+edgUzO9vMvjSzYuDfwfunBO/PCD5+e/C5PzOzQ/d2rL0c/xQz+yL4+C1mdmdwQZzar/MMM1sY3G+9mf3BzI4NHmdSY98/561mlw0MqOM5rzKzd8xsU/D92mJmz5pZ79qvJfjrhTUvb9U61rHBY+UGX8MiM7uisXVLdNEZhUQtM4sD3sKbGvsZ4D4gA7gM+MzMjqjx7TkeuBF4GZiBt+DMaOASYLyZHeScK6v1FKcDvwAeBB7Cm9ywpreBHcAf8RakuR54w8x61/PM4yTgquCxnwB+AtyAN1HfX2u8zrOB/wCr8SbzqwAuBE6tx3OEZGbtgfbA9jruvgFveux78RbWGQZcChxjZsOdczl4r38i3vv/CfBIHc9xefA1zsabNbgQ+DHwoJn1c87d2NTXIT7ze1ZC3dreDTgKb6bMG/ax33XB/Y6vtT0db6baD2tsM2rN0hncfknwGGfV2NY7uK0c2L+Ox0wJ3v9Are0/D26fXMexbq1jWyHQu1aNS4AtNbbF4s0yug1oX2N7Kt5qdA6YVI/39Nbgvj/CW42tC3AY8EFw+x11PCaljm0/Cu7/61rbHTCljv274l16e76O++7BW0a0n9//5nRr2k2XniSanQ8sA740s8zqG97Zw7t4ZwpJ4LUVOOeKYc+6xu2C+84MHuvQOo7/unPumxDPf3et36uP9YPLOHvxinNuXfUvzvvr+QHQxcxSg5sPArrh/RHeVWPfArxv6Q31Ht5ZwBbgU7wlaf9OHWuYO+cKYc9lu4zg+7UQb3rtut6vupwJJACP1/yMgsd6De/y9o8a8TokiujSk0Sz/fF6FO0IsU8msAHAzM4CfoW3gFJcrf3a1/HYFXVsq2lNzV+cczlmBnWvi7zPxwfVXFu5AOgT/L2uldP2tppaKFfjva5k4Gi8S2vtnXM/6JVlZsfg9ZI6FEisdXdd71dd9g/+fC/EPp3reSyJUgoKiWaGtwjM9SH22QFgZj8DXgDmAr/EC48SIAavnaOus+eiUE/unKsMUVd97O3xNY9R32PV11z3XbvNq2a2DfibmX3lnNtzhmJmo4F38JZFvQmvS20x3iWmqdS/o0t1/RfgncXUpa7AlBZEQSHRbCWQBcx0zlXtY9+JeMFwtHNuTwCY2eAI1hcOa4M/B9VxX13bGuouvHaaP5vZ88656gb7CXgheqJzrroGzCyF+p9NgPcZAWQ750KdVUgLpjYKiWZP4zXK1nlGYWY1L2lU4n0bDtS434CbI1lgGMzD+yY+KdhDCYBgG0aTu5c658rxelh1xLsMVa36bKf2Gc3vqPvvQgHQoY7tL+It+3lbdXtRTcG2j4SG1i3RRWcU4qcfmVnta+PgfTt9CK/XzI+BO4PX02fidWHthddAWoJ3HR68xezPAGaa2dN4bRSn412rj1rOuYrgVCbPAXPN7HG87rGT8Noz+uAFYFM8g9cWcb2Z/ds5l4e3fvZ1eN19HwHK8N7rEXjjLmqbDRxrZr/B63HmnHNTnXMbzexKvLXSvzGzZ/BGq2cBw/E+gyHAuia+BvGRgkL8dELwVtty4CHnXLmZnYw3FmEi3hgDgM14bRFPVT/AOTfVzNLw/vj9A2+swmt4199ziGLOuefNrALv7Oc2vK6yjwOLgOl4bQdNOX6Fmd2O14vqWuA259xnZnYG8HvgT8HneA84Evi4jsNcBdwP/C+QFtw2NXj8J81sBd64jMlAO7ywWR48/tam1C/+M6/HnohEGzP7FV7ojXXOzfa7Hmm7FBQiPjOzeKCyZi+rYBvFIrzBhd3cD0eVizQbXXoS8V9f4E0zm4rXC6or3hQefYArFRLiNwWFiP924DUWnwd0wmvMXgzc5Jx70c/CRECXnkREZB80jkJEREJSUIiISEgKChERCUlBISIiISkoREQkJAWFiIiE9P+eK5kLg7XvFAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_CEL.lr_find()\n",
    "learn_CEL.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### training epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T17:11:48.973219Z",
     "start_time": "2020-07-01T16:56:32.666300Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>seq2seq_acc</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.456067</td>\n",
       "      <td>2.653436</td>\n",
       "      <td>0.612640</td>\n",
       "      <td>0.433706</td>\n",
       "      <td>01:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.041358</td>\n",
       "      <td>2.261911</td>\n",
       "      <td>0.642574</td>\n",
       "      <td>0.462979</td>\n",
       "      <td>01:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.675195</td>\n",
       "      <td>1.926699</td>\n",
       "      <td>0.680742</td>\n",
       "      <td>0.488075</td>\n",
       "      <td>01:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.384582</td>\n",
       "      <td>1.713956</td>\n",
       "      <td>0.702641</td>\n",
       "      <td>0.511958</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.127888</td>\n",
       "      <td>1.588813</td>\n",
       "      <td>0.723198</td>\n",
       "      <td>0.536749</td>\n",
       "      <td>01:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.813250</td>\n",
       "      <td>1.529009</td>\n",
       "      <td>0.734520</td>\n",
       "      <td>0.553977</td>\n",
       "      <td>01:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.497641</td>\n",
       "      <td>1.541128</td>\n",
       "      <td>0.743082</td>\n",
       "      <td>0.570197</td>\n",
       "      <td>01:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.262595</td>\n",
       "      <td>1.580004</td>\n",
       "      <td>0.747232</td>\n",
       "      <td>0.581183</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.140268</td>\n",
       "      <td>1.620333</td>\n",
       "      <td>0.750187</td>\n",
       "      <td>0.587652</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.086930</td>\n",
       "      <td>1.639049</td>\n",
       "      <td>0.750771</td>\n",
       "      <td>0.589219</td>\n",
       "      <td>01:32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1cH/8c+ZJSvZCFsg7FZcUMQKtYobSFuXutDFurRqRX8+3dxaa/u4UJdK7fK01m5aKdhV2qrVWrVacUHBrSKLgCKbQCAkkMmeTGbO74872SdkJpnkzgzf9+s1zMy9d+49czJ8c3Lm3HONtRYREUl9HrcLICIiiaFAFxFJEwp0EZE0oUAXEUkTCnQRkTThc+OgxpgWnF8m1W4cX0QkReUDYWtt1Ow2bgxbNMaEAZObl4/PYwb9+CIiqSgQCABYa23U3hVXWuhAtcnMLXh4+QbOOrrEpSKIiKSWwsJCAoFAjz0brvahW3RSk4hIorgb6MpzEZGEcbmFLiIiieJyC12RLiKSKBqHLiKSJtwa5QJAWC10kaRgraWiooLGxkbC4bDbxTmoeDwefD4f+fn55Obm9mtfrga68lzEfdZadu7cSU1NDZmZmXi9XreLdFAJBoPU19dTVVVFXl4eo0ePxuPpW+eJAl3kIFdRUUFNTQ0jR45k6NChbhfnoBQOh6msrKSiooJAIEBRUVGf9qNRLiIHucbGRjIzMxXmLvJ4PAwbNoyMjAxqa2v7vp8EliluGuUi4r5wOKxuliRgjMHn8/XrOwy10EVE0oS7wxaV6CIiCeNqoGvYoohI4qjLRUTS1sqVK1mwYAFVVVUJ37cxhgULFiR8v/2hyblEJG2tXLmS733vewMS6CtWrGD+/PkJ329/uDsOXW10EUkC4XCYUCiE3++P+TXHH3/8AJaob9RCF5G0tGDBAq677joAJk6ciDEGYwxbt27FGMO1117Lz372Mw455BAyMjJ45ZVX2l43Y8YMioqKKCgoYObMmSxdurTb/rt2uSxYsABjDOvXr+eCCy4gPz+fkSNH8uUvf7n1SkMDzuUWuogkq5ZQmLJAo9vFaFNSkIXPG3sbdP78+QQCAX7605/yyCOPUFLiXB2t9f6vf/0rY8aMYeHChQwZMoTJkycDsH37dr761a8yduxYWlpaWLZsGRdddBE1NTVcccUVvR533rx5XHDBBVx55ZWsXr2a7373uwAsWrQo3rccN1cDXU10keRVFmjkpHuWuV2MNi/feBpjh+bEvH1paSnjx48HYPr06UyYMKHT+sbGRp577jny8/M7Le8YvOFwmDlz5lBRUcF9990XU6BfddVVbX8ZnH766XzwwQcsWrSIBx98EGMG9hrKLs+26ObRReRgNmfOnG5hDrBs2TIWLlzIqlWr2Lt3b9sZ7VlZWTHt95xzzun0/Oijj6axsZE9e/YwatSo/hf8AFyenEuJLpKsSgqyePnG09wuRpuSgtgCNeb9lXS/QP3KlSuZO3cus2fP5he/+AVjxozB7/fzq1/9KuYuk+Li4k7PMzMzAecvgoGmPnQRicrn9cTVxZFqonV/PPzww/j9fp544om2IAZobm4ezKL1mUa5iEjaag3lhoaGmLZvnSCr43zk5eXl/OMf/xiQ8iWazhQVkbQ1depUAO677z5WrFjBm2++ecDW9llnnUVtbS0XX3wxzz33HH/4wx846aSTGDly5GAVuV80fa6IpK2TTjqJm266iUceeYRZs2YxY8YMdu3a1eP2c+bM4f777+ftt9/m7LPP5o477uDaa6/l4osvHsRS951xI1SNMVUmM7fg/mdXM/+kSYN+fBFpt23bNoC2IX7int5+FoWFhQQCgYC1tjDaes22KCKSJvSlqIhImtCXoiIiaUItdBGRNOFyC12JLiKSKGqhi4ikCXcvEi0iIgnj7rBFTbcoIpIwMQW6MWa6MeYxY8wuY0ydMeZdY8xNxpjM3l/dM8W5iEji9DrbojHmMOBVYCNwLVABzAbuAo4AvtTXg6sPXUQkcWJpoX8ByAI+Y61daq193lp7M/An4EJjTOxXVe1Co1xEJFVcdtllna561Hpt0sWLF8f92oESy3zowch916ucBiLrQn09uFroIpKqSkpKWLFiRdu1SJNBLIH+e+B64FfGmBuBSuA04FLgx9bacNcXGGOqetlnAagPXURSV2ZmJscff7zbxeik1y4Xa+124Hic/vLNOC3zx4B7rbW39OvoaqKLyAB59NFHMcbw4osvdlt355134vP52LVrF88++yznnHMOY8aMITs7m0MPPZTrr7+e6urqA+6/py6XRYsWMWXKFDIzM5kyZUrMl65LhFi+FB0PPAHsBs4HqoBTgO8YY8LRQr2nqR077LMKKNCoRZEkFmqB6p1ul6Jd/hjwxn7VzLPPPpthw4axePFiTjnllE7rHnroIebOncvo0aN5/PHHmTVrFldddRV5eXm8//773H333bzxxhu8/PLLcRXxwQcfZP78+Zx//vn85Cc/Yf/+/dx2220Eg8FOV0EaKLHUzkIgD5hurW29jtMLkevx3WqMedBau7UvB9eXoiJJrHon/Oxot0vR7prVUBT7nO1+v5+LLrqIRYsWcd9995GbmwvAK6+8wvvvv8+dd94JwNVXX932GmstJ554IlOmTOHkk0/mnXfeYdq0aTEdLxwOc8sttzBz5kz+/ve/t12z9IQTTmDKlCmMGTMm5rL3VSy/MqYD73YI81ZvRl5/WF8Prh4XERlIl19+ObW1tfztb39rW7ZkyRKKioo499xzAdi9ezff+MY3mDhxIllZWfj9fk4++WQANmzYEPOxNm7cSFlZGRdddFGnC1BPmjSJE044IUHv6MBiaaHvAqYaY3KstfUdln88ct/nv8mU5yJJLH+M0ypOFvnxt3CPOeYYpk2bxuLFi7n00ktpaGhg6dKlXHzxxWRmZhIOh5k7dy7l5eXceuutTJ06ldzcXD788EPmzZsX88WlASorKwEYNWpUt3UlJSVtVyMaSLEE+r3Ao8Azxpif4nwpeipwI/CctXZNXw+uFrpIEvP64uriSFaXXnopN9xwA1u3bmXFihUEAgEuu+wyANasWcPatWtZsmQJX/pS+zmSgUDXUdq9Ky4uBpwWf1dlZWV9K3ycYhnl8hgwF2gCfgn8A+fL0TuA8/pzcPWhi8hAu+SSS/D5fCxZsoQlS5Zw5JFHMmPGDIC2rpGMjIxOr3nggQfiPs6UKVMoKSnhT3/6U6flmzdv5tVXX+1j6eMT01fG1trngOcSfnTluYgMsOHDh3PGGWfwm9/8hvLychYuXNi27vDDD2fSpEncdNNNABQUFPDnP/+Zt956K+7jeDwe7rjjDubPn8+8efO44oorqKqq4tZbb6WkpCRh7+eAZRiUo/RAF4kWkcFw+eWXU1ZWhjGGSy65pG253+/n8ccfZ/LkyVx55ZV88YtfxOv18pe//KVPx7niiiv47W9/y9q1a5k3bx633347N998M7Nnz07UWzkgY10IVWNMlcnMLbj9b69z89lHDPrxRaRd65d148enfn95quvtZ1FYWEggEAj0dK6PLhItIpImXA30kE4VFRFJGFcDvSXcbV4vERHpI7XQRUTShLst9JACXUQkUVzuclGgi7jN4/EQCvX5OjWSQOFwuNM8MPFSoIsc5Hw+H8FgkLC+03JVKBSisbGRrKysPu/D5T50fYBE3Jafn084HKayshI3zksRJ8xb53vJy8vr835iny1+AKgPXcR9ubm55OXlUVFRQXV1NT6fq7Fw0AmHwzQ1NWGtZeTIkWRnZ/d5X+4GurpcRJLC6NGjCQQC1NbWqutlkHm9XoYOHUpeXl6/whwU6CKC88VoUVERRUVFbhdF+kF96CIiacLVQA+qD11EJGF0pqiISJrQOHQRkTShPnQRkTShuVxERNKEulxERNKEvhQVEUkTLg9bVB+6iEiiqIUuIpIm1IcuIpImXB7loi4XEZFEcTXQG4Ihzb8sIpIgrgZ6YzBMU4ta6SIiieBqoAM0BnUtQxGRREiCQFcLXUQkEVwP9KYWtdBFRBLB9UBXC11EJDFiDnRjzKnGmH8bY6qMMfXGmHeNMVf1twDqQxcRSYyYrilqjLkUeBB4APgJEAQOAzL6WwAFuohIYvQa6MaYscCvgO9aa+/psOo//Tmwidw3atiiiEhCxNLlckXk/ucDUQC10EVEEiOWQD8ZWA/MM8ZsNMaEjDE7jDELjTFRu1wi/ew93oACE2mi68QiEZHEiKUPfXTk9nPgFmAdMBv4DjAWuLg/BVALXUQkMWIJdA+QB1xorf1LZNkLxphs4JvGmNustZs6vsBaW3igHRpjqgwUADQp0EVEEiKWLpfKyP0zXZY/Fbk/tk9HjvS5aBy6iEhixBLoa3pY3jpQpU+J3DbKRS10EZGEiCXQH4ncn9ll+ZmABd7oTwEadeq/iEhC9NqHbq192hjzFPALY8ww2r8UvQb4tbV2W18O3NpCb1KXi4hIQsR0pijwOeB7wI3AcGA7cDNwz4FedECRRFcLXUQkMWIKdGttHfDNyC0h2vvQ1UIXEUkEF2dbbB3loha6iEgiuBforV0uaqGLiCSEa4He9qWo+tBFRBLC9QtcaJSLiEhiuN5C1ygXEZHESII+dAW6iEgiuNhCdxK9QYEuIpIQ7gV6pIVe3dDiVhFERNKK633ogYYg4bB1qxgiImnD9VEuAJv21rpdBBGRlOdaoHs8pu1xRW2TW8UQEUkbrne5AITU5SIi0m+udrnkZTlzg9U16YtREZH+cjXQczOcQH/xvQo3iyEikhZcDfTd1Y0A5GZ43SyGiEhacDXQzztmNAD76prdLIaISFpwNdBH5GcBsFejXERE+s3VQB9d4AR6WaDRzWKIiKQFVwN9WF4moHHoIiKJ4GqgF+c6gV5VHyQY0rzoIiL94XIfembb493qdhER6RdXA33c0Jy2WRc/3F/vZlFERFKeq4Hu93oYNsRppW/eW+dmUUREUp7rsy2WREa6vL5ln8slERFJba4H+tihOYBGuoiI9JfrgX7i5GEArN4R0KyLIiL94HqgH11aAEBtUwvlNRrpIiLSV64H+oRhuW2P7/znehdLIiKS2lwP9CGZvrbHT64pc7EkIiKpzfVABzj76BIAPEYXuxAR6aukCPTbz50KQNhqPLqISF/1KdCNMQuMMdYYsyoRhSjK8VOY4wfgvT01idiliMhBJ+5AN8YcCXwb2JOoQhhjOGT4EAC279MUACIifRFXoBtjPMCDwG+BDYksyKjIGaOapEtEpG/ibaFfB5QC/5vogowpygZgc0VtonctInJQ8PW+icMYMwm4HbjYWlttWqdJjL5tVS+7K+i6YPrYIgDe+TBAYzBEll8XjhYRiUdMLXTjpPcDwDPW2scGoiAzJjiB3hwK88ZWTdQlIhKvWLtcrgSOA74ey8bW2sID3YBA19cUD8nk0JHOF6O3/WNdjMUSEZFWvQa6MWYYcA9wN1BnjCk0xhTidNd4I8+zElGY86aPAWBzRR31zTrBSEQkHrG00Etx+rzvBvZ3uJ0ITI08XpCIwlxw3Ni2x79YtikRuxQROWjE8qXoJuC0KMt/CgwB5gPbE1GY4iHt1xhdtmEv3/rkYYnYrYjIQaHXQLfW1gIvdF3eOpLFWtttXX9865NT+OEzG3m3rJpAfZCCyBmkIiJyYEkxl0tHsw8b0fb48sWvu1gSEZHUEvM49K6stacmsBxtDhuV1/a4sq55IA4hIpKWkq6FbozhmjkfASBbJxeJiMQs6QIdYNpY50TSzXvraAmFXS6NiEhqSMpAP2S40+3SHAqzTbMviojEJCkDvbQomyy/U7Q5P37R5dKIiKSGpAx0j8e0TdYFsGO/WukiIr1JykAH+O2lx7U9/udqXTxaRKQ3SRvouZk+vvTx8QA8vmqXy6UREUl+SRvoAOdMGw3Au2XVPLRiq6tlERFJdkkd6MeOa+9Hf2jFNhdLIiKS/JI60D0ew1dOnQzApvJaXtlU4XKJRESSV1IHOsA1p3+k7fEXH3wNa62LpRERSV5JH+iZPi83zD0UgLCF65e+43KJRESSU9IHOsDXZh/S9vjRt3dS16SrGYmIdJUSgW6M4brTD217fte/1rtYGhGR5GTc6JM2xlQVFBQUVFVVxfW6CTc92fb4qDEFrNkZ4JLjx3HneUcluogiIkmnsLCQQCAQsNYWRlufEi30Vkv/38fbHq/ZGQDgDyu3c/3Dq9wqkohI0kipQJ85cWjU5Y9E+tWrG4M0BkODXCoRkeSQUl0uAIH6INNu/zcAnzpyFE+v291tm9ULPkF+lq5FKiLppbculz5fgs4tBTl+ti48q+15x371Vn96bTtXnzJ5MIslIuK6lOpyiWbRZcd1W7bwqQ1c9dCbLpRGRMQ9KR/osw8byd3znFEu3/7UYW3L//3uHibc9CSrd/TQrVOzB9Y9CnWaTkBE0kPK9aH35ht/fpvH3+k83e7d847iwpnjOm/434fg8a87j0dOhYknO7fxJ0BWQcLLJSLSX731oaddoAOs3Rng7J8v77Z8y91nYoxxnrz0I3jph9DS2KVwHhg93Qn3CSfBuOMhI3dAyikiEo+DMtABGoMhZv1gGRW1TZ2Wv3zjaYwdmuM8CTbCjjdgy0vObeebEO4yrYDHD6Uz2lvwpceBL3PAyi0i0pODNtBb7apq4ISFz3dbXpybwa2fPoL65hBfmDHWabk31cL2lbDlRSfgy94ButSPL9tptU88GSaeAiXTwJtyg4VEJAUd9IEOsGZHgE/f170LpqOXvnUa44pzOi9s2A/bXm1vwZe/2/2Fmfkw/sT2FvyII8CT8t81i0gSUqBH1Da1cPXv32J5LxfJWPmdOVxw/wq2VdZzeEk+R47OZ/u+eu67cDojPNWw9eX2gN+3ufsOcophwqz2FnzxIdDaby8i0g8K9A6CoTCLlm/hxEOG8dCKrSx9c0dcr99wx6fI8nvbF1R92Dngq3d2f1FeSXvrfeLJUDiu+zYiIjFQoMdgwePrWPzq1l63mz6ukEe/cmL0ldY6LfbWcN/yEtRH+WugaEJ7633CSZA3sl9lF5GDhwI9RuGw5fT/e5HNe+uYOCyXZd88lbe37+fXL37AM+v2tG33k89PY96xpb3v0FooX98e7luXQ1Og+3bDD3O6aEpnwtgZUDRRXTQiEpUCPQEamkMcfuvTbc+Xf/s0SotyDvAKaAmFWbMzQDBknVkiwyFn1ExrwG9fAcH67i/MGeYMkyw9zrkfcyxk5iX6LYlICup3oBtj5gBfBD4OjAX2Aa8Dt1lr1/SlUKkW6AB/fG0b//vo2k7LfnrBMRw/qZgXNpZz0yMHropnrzuZj4zsEMwtzbDzrfZw3/kWNFV3f6HxOCNnSme034oP0UgakYNQIgL9r0AxsBRYD4wEbgSmAqdaa1fGW6hUDHSA0370Alsq6vq1j7duPp2huRntZ6y2CoehYqNzotOON+DDN2DvBrqNgwfIKoy04Gc692M+CtlRf74ikkYSEegjrLXlXZYVAluA5621n4m3UKka6BB9ut7++PUlx9LUEuaUQ4dTmJPReWVjwGm573gTPnzdCfrGaHVmYPiUDiE/w3nu8UbZVkRS1YD1oRtjXgestfZjfXhtygY6ONMKPLW2jD+u3E5VQ5CK2ibuOHcqx44vwlrLP1eXceHMcRRk+1n+fgWXPPhazPv+93UnM6ogK/oFOqyFyk2dW/Hl68CGu2+bkQelH+3cVZMT/YpPIpIaBiTQjTHDgW3An621V0RZ31tSFxQUFJCqgd5XD7y0mbv+tT6mbb94/HjOPWY0WyvrWfFBJe+WVTOmMIsff/4YCrI7hH1TLez6b3vA73gj+nBJgKGTYezM9pb8iCM0bYFICkl4oBun8/cR4AxgurW2W0Ip0HtW29RCeXUjI/KzWLczwAX3x/0VBKdNGU5+tp8b5k7pPl2BtbB/i9NNs+MNp6tmz9ruk44B+HNg9LHOcMnSGc4QysJx4NXl+0SS0UAE+o+AG4DLrbWL+1KoVO9yGQjBUJjP/XoFqz6Mr07+fd3JHDqyl2GNzfVQtqpzV01t92uxAmC8TqgPnQRDJ0buI7fC8eDPiqt8IpI4CQ10Y8xdwHeBa6y19/a1UAr0A9tb08SMu54DoKQgix985mgyfB6efXcPDy7f0m37b8z5CNfPPTT2A1gLgR2w4/X2L1zL3oFwsJcXGigodc527Rj0reGveeNFBlTCAt0YcztwC3CjtfaH/SmUAr3vNpXXcOeT63lh495u6zJ8Hppbwlx9ymSmjytk9mEj8HtjHK8eCkLgQ9i3xZnCoO1+M+zfCqGmXnfBkJGdA771cdFEDasUSYCEBLox5jZgAXCLtfbO/hZKgd5/wVCYn//nfe59ftMBt7v5rMO580nna47Hvnoix4ztQ7CGw1Czqz3g225bnftgDGPzs4dGb9UPneTMUKnpDkR6lYhx6DcAPwL+CdzVZXWTtfbteAulQE+ceIdFdvTObZ+gINtPQ3MIY8DnMfhibdG3shZqyzsH/f5I675yc/T5a7rKzG8P94KxkDsMcoc70yDkFkfuh6lLRw56iQj0F4BTeli9zVo7Id5CKdATy1rLa1v2sbemiUnDc/nrmzvYsb+e59aX9/7iLjwG3r/rTLyeBLSYrXUuEtIW9ls6B39Pwyt74st2gj2nOHI/7ADPhztz4KjlL2lEk3MdxPZUN/Kx7/+HQ0YM4VufnMIvX/iAd2IYRfPZj5byo89NG/gCNla3t+ZbW/TVO52gr6t07kPNfd+/N6N7Kz/q88gvgaxCzZGTbqx1PkMtTc73RKGmyOMelrUtb46yLOgM/7UhZ7K9cKj9cdf7bsvCnV/busyGnOXdloU6bN++rPCWTQQawwp06ey4O5+lovbAYTm+OIeWkGVnVQOj8rPYXd0Ydbv/OXUyN35yCis37yM/28eRowsSU0hroammc8DXVXS4r+zwPLI+2gyWsTLe9tZ+xhDwZztj9Tvdd3ickdt9WU/bezPS+6+FUEsk/JqciedCzd3DMNQalF3X9bRtMMagPcD6/jQIklDhwmoCTSjQpbtw2GIMbROFNbeEOeNnL/HB3v5NQPalj4/ntk8fmZhum3g113UP+QM9b64ZnHIZb/RfCgda5s1wpnWwkRZa2+Ow88uu0/Notw7bhLu+PpZ9WOe4nUK4Y9B2COlo00+kEuMBb6ZT574M57Evw3nuzQBfZJ3H6/wsPV7w+CKPPe3LTGR5t2Wt23s6L2tbF9uywlmXE6ipV6BLbOqaWjjytmcSsq9l3zyVicNyaQyG2F/fjMFgsZQUZCdk/wkRbHRa+h1b/fWVzi+GYEPk1vq4vsOy+u7LmuuIOjumtPP42kOyLSj9kTD1twdn2/qM7kHbtl200O24LDNKOGdG30+KTIGhPnTpk3DY8tTa3Xxk5BCGDclka2UdR48pINAQJNAQxAIZXg8+r+GHT2/kkbd3UpDtJ9DQ28lJMK20gL//zwnxj6hJdq39tZ1+GXQN/p5+KUTZLhR0WmbGRFppnh5u5gDrOq7vaR+9vL7HoI0Slm3b+rsEcUZ7C1f6TIEug6qpJcT1D7/Dk2vKYtr+iJJ8Lpw5ls/PGMvb26vI8Hk4dlzRAJdSJDUp0MUVT6/dza9e2MQ7O9rHoR9dWsDqHTGMSwc++H6Chk6KpBEFuiSdX76wiXue3tjrdu/deQYZvjTrlhHpBwW6JLVdVQ1c+5dVvL51X7d1p00Zzu8un+lCqUSSkwJdUoK1FmMM1lpm/WAZO6sa2tbd89mjObq0gMNG5Xd6TThs2VJZx6Rhud2v0SqShhToknKCoTBn37ucjXuijxGfVlrQqW8e4NWbZjO60BkOWV7dyHPryzlj6ig8HkNTMMSm8lrKa5ooyPbz9Nrd5GX5+NTUUXx0fJF+GUjKUKBLStpX18wVS97g7e3xfUaOHVfIf+N8zdDcDPbVNfPjz02jIRhi0fItfOfMw5l7xMi49iMy0BToktKstfx3exX3PL2B17Z072cfKBleD8u/fRoj8nWFJkkeCnRJO6397a3W7gxw9s+Xd9rm88eVsvTNHZ2W/eKiY8nweZg+rpBr/vI2r2yqPOBxTp0ynN9dNkNdMpI0FOhyUHBa8vvxGMPhJflk+eM7I7Gytonfr9xGVX2Q/Gw/9/7nfQCuO/1Qrjn9IwNRZJG4KdBF4hQKW2b/+AW2VbbP3PjE12ZxVGmCZpEU6aPeAl1nbYh04fUY/vWNk5g5cWjbsk/ft5z7nn+fUFiTb0nyUgtdpAfBUJjbHl/Hn17b3rYsP8vHH+Z/jKNL2xtI1lre2raf5lCYvTVNfLC3jlA4zLxjSynM9rO+rIaPTy7ucSqDdbsCZPo8ZPm95Gf7yc/yD/h7k9SkLheRfvrXmjK+8sf/9ns/p00Zztpd1RTnZvDhvnrqmkMH3P6yEyZwy9lHaE4baaNAF0kAay2/e2Urt//z3UE97vxZE7n57CMG9ZiSvBToIglUVd/MgsfX8diqXYAzXr05FOau86dy2Kg8RuZnMaYwm+376vndK1tZ/OpWpo7JZ+3O6qj7mzomnwyvhyNG5zMqP4v39tSypaKONTvbz4S9e95RXDhz3KC8P0luCnSRFBSoDzLt9n+3Pf/n12cxdYxG2RzsNMpFJAUV5Ph5/X/ntD3/3K9X8NjbO3GjASapQ4EukqRG5GXx5DdmMbogi4ZgiGsfXsV5v3yV17fsw1pLWEMopQt1uYgkuZ1VDdywdBUrN0efyyYnw0tOho9Mn4fPHDuGM48uoaE5xOtb9lFe08TrW/YxIi+Tw0ry2Li7lr21TRxTWkBOpo8vnziR4XmZg/yOpK/Uhy6SBqy1LNtYzk+fez/my/jFq6Qgi4raJkJhS7bfS2FOBiUFWdzwiSl8fHLxgBxT4qNAF0kj1lo2ldfy7Po9bN5bR0lBFiPzs9hSUccrmyrYsLv7HPLGgLVwyIghZPk97A40MjQ3g/f21MZ0zGy/l4f/3/GdTqYSdyjQRQ4idU0trNtVjddjmDw8l7wsf48nJu2taWLF5kpWfFDBW9v2M3VMAaMLsmkIhphQnEN9c6oQSt8AAAmESURBVIgHXt5MRW0zuRleFpxzJJ/9aKlmn3SRAl1E+mxTeS0X/3Yle6qbADiiJJ8zpo7i7GmjmTgs1+XSHXwU6CLSL/vrmrnpkdU8s25Pp+X5WT4OL8lnTFE2U0cXMCI/k5rGlrbrvxrAo2kLEkqBLiL9Zq1lxQeVPLmmjKfW7mZfXXNMrxs2JIOhuRkUZmeQn+2jJWzZvq+evEwfZYFGfB5DXXOIQEOQUflZfGTkEACaWsIMG5LBsCGZtIQtE4pzuPSECWT64pvnPt0o0EUkoYKhMG9u3c/6smr21TWzYXc17+wIsK+umSyfp9dJx/pq3FAn1E88pJhDR+QdlK3/hAS6MWYI8H3gc0AhsA643Vr7eF8KpUAXST+tWbK1sp5N5bXsr2+muiEIQE1jC9WNQYKhMIXZGWT6PBTlZtDcEgageEgG5dVNVNY1E7aWUNhS3xxid6CB6sYW/rt9Px2jamR+JidMHkZRTgbFQzIYNiSDCcW55Gf7I0Mu/eRk+GgIhmhoDuH1GLL8znmUuwONZPg8FGZnkJPpxecxWAsWkn5my0QF+rPAscCNwBbgMuBi4NPW2n/FWygFuojE44O9tfz6hQ948b29lNc0JXTfXo8hbC3WQpbfQ16Wn/wsH/nZfudsXGMItoTJ8nsYkZfF4SX5TBtbwNDcDDJ8HjK8Hufe5yHT6yXD58EYyPR5Ej4iqN+Bbow5E3gSmGetfTSyzAAvA8XW2sPjLZQCXUT6wlrL5oo6nlm3mw1lNW1/BeysaqSiNrFB31/GgN/rBL7fa/AYg8dj8BjwGoMxBm/kees6cC6B2BIOEwo5F0P3ew2ZPi/GwPPfPYtgQ22Pge6LoVznAwHgH60LrLXWGLMEuN8Yc4S1dnAniRaRg5IxhsnDh/CVUw/ptq65JUxNY5DGljD765ppCIbI9nvJ8ntpaA7REg7T3BKmdGgO1lr21wVpbAlR3xzC73ECta6phZrGFqrqm6lqCJLp82Kx+DyGxmCYXVUNrPqwivf21NDbVDrWOmVq7VZKhJZeDhpLoE8F3rXWdi3V6o7rO64wxvTW9C4IBAIUFurMMxFJXbbtn9Y7G2VZ+8Y9x3HXNabL2sjem+oA8nvaSyyBXgy8F2X5vg7r+yQQCAzMpBTpqXUybNVZ7FRn8VOdxW8w6ywf6LHJH0ugw4F+sURZ11P/TqvWFnxv20k71Vn8VGfxU53FL5nqLJb50CuJ3gofGrmPPqeniIgMqlgCfR1wuDGm67ZHRe7XJrZIIiLSF7EE+qM4JxN9usvyLwEbNcJFRCQ5xNKH/i9gGfCgMaYY58SiS4FZwLkDWDYREYlDr4EeGXN+Hs6p/9/Haa2/i3Oi0RMDXD4REYmRa5NzQXJ8K5wqVGfxU53FT3UWv2Sqs1j60EVEJAW40kIXEZHEUwtdRCRNKNBFRNKEAl1EJE0MaqAbY4YYY+41xpQZYxqMMW8aY84ZzDIMNmNMqTHmZ8aY5caYWmOMNcac2sO2Fxlj3jHGNBpjdhhjFhpjsqJsN9IYs8QYU2GMqTPGvGyMOaE/+0wWxpg5xpjFxpiNxpj6SJkfMcYcFWXbucaYlZHPUrkx5jfGmG4jDeL53MW6z2RijDnBGPOMMWZn5Oe81xjzvDHmjCjbqs56YIxZEPn/uSrKutSoN2vtoN2AZ3HmhrkCmA08BISAMwezHIP8nk8FyoGnceaUt8CpUba7JLLul8BpwFeAGuAvXbbLwpluYStwIfAJnJO/GoDpfdlnMt2AvwLPA1cDpwCfB94EGoHju9RrMLL96ThnLpcBrwCevnzu4tlnMt2As4B7gS9E3sP5wFORn/0XVGcx1eGRkf9Du4FVfX2PbtfbYFbYmZEP2PkdlhlgObDe7R/oAL5vT4fH5xEl0AFv5If5jy7Lr4xs/7EOy74SWXZsh2WZwGbgqb7sM5luwIgoywqB/cDfOyx7HXi7S/3Ojby3C/ryuYt1n6lwwzlp8EPgedVZr3XlAVYCPwdeoHugp0y9DWalPQBU0f03WmvAHOH2D3YQ6qCnQD8xsnxel+U5QDNwT4dlzwKro+z7LqAFyIt3n6lwi/wHeC3yeEzkvV0fZbsdwNJ4P3fx7DNVbjh/yT2jOuu1nm6IlDe/a6CnWr0NZh96LFc+Oli1vvdOM1daa+uBD+hcN1O7bhexGqdVfniH7WLdZ1Izxgyn8/uO+t4i1tC9vmL53MWzz6RkjPEYY3zGmNHGmO8BhwL/F1mtOovCGDMJuB34mrW2OsomKVVvgxnoxUSfO73fVz5KA63vvaf6Ke6ybSz1GM8+k5YxxgD343xWfxRZrPqKbilOv+xO4Frg89bapyPrVGddRD5bD+D8FfNYD5ulVL0N9rDFuK58dBDqqQ66Lo+nHmPdZ7L6IU5X1dXW2vVd1qm+OrsRmAmcg/NF+VJjzIVdtlGdtbsSOA74egzbpkS9DWag68pHPauM3PdUP/u6bBtLPcazz6RkjLkLp3/zGmvt4g6rVF9RWGs3W2vfsNY+Ya29EHgG+EXk4jSqsw6MMcOAe4C7gTpjTGFkyKAP8EaeZ5Fi9TaYga4rH/VsXeS+U9+ZMSYHmEznulnXdbuIo3CGR23owz6TjjHmduC7wI3W2nu7rI763iKOont9xfK5i2efqeJ1oAgYjuqsq1KcizvfjTOCqvV2Is772Q8sINXqbRC/ST4L50+Jc7ssfwnY4PY33YNUBz2NcvHhDDF8tMvyKyLbdxx//dXIsmM6LMvA+aLz6b7sM9luwG2RMt58gG3eAN6i87CvOXQfex3z5y7WfabCDWe43H9wgsmnOutWP0NwxoJ3va0CNkUeT0q1ehvsD9jzQAXwZZwTXRYDYeDTbv+AB/i9fzZy+0HkB3Zb5PkZHba5NLLuvsiH6X+AauCvXfaVhXOBkc3ABThjV/+Jc1LER7tsG9M+k+mG08VigSeA47vcpnfYbjbOMM2HI/8RvgjswhlP7O3L5y7WfSbbDfgjzsVnPoNzMtYXaD+x6Guqs7jq8gW6j0NPmXob7MrKj4TLbpwz//4LnOf2D3EQ3rft4ba1y3aX4AxbasIZqXAPkB1lf6OA3+P0tdXjnLgwq4djx7TPZLlF/kPFWl+fAl6LfJb24oxYKOrP5y7WfSbTDfgasAKnb7Ylcv9M1xBRncX8+VsVZXlK1JvmQxcRSROabVFEJE0o0EVE0oQCXUQkTSjQRUTShAJdRCRNKNBFRNKEAl1EJE0o0EVE0oQCXUQkTfx/HIhEjmLP2AwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_CEL.fit_one_cycle(10, 5e-4, div_factor=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T17:55:57.169355Z",
     "start_time": "2020-06-22T17:55:55.635574Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# export model \n",
    "learn_CEL.export()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Initialization Cell",
  "kernelspec": {
   "display_name": "CapGen",
   "language": "python",
   "name": "capgen"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "293.292px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}