notes/nlp/nlp_Bigrams-Language-Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package words to /home/yui/nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/yui/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/yui/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package brown to /home/yui/nltk_data...\n",
      "[nltk_data]   Package brown is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import pairwise_distances\n",
    "import pandas as pd\n",
    "import math\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from nltk import word_tokenize\n",
    "nltk.download('words')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from tqdm import tqdm,tqdm_pandas\n",
    "nltk.download('brown')\n",
    "from nltk.corpus import brown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = brown.words(categories=\"news\")\n",
    "sents = brown.sents(categories=[\"news\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words:  100554\n",
      "Number of sentences:  4623\n",
      "Number of unique words (including capitalized) 14394\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of words: \",len(words))\n",
    "print(\"Number of sentences: \",len(sents))\n",
    "print(\"Number of unique words (including capitalized)\",\n",
    "      len(set(words)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', 'Friday', 'an', 'investigation', 'of', \"Atlanta's\", 'recent', 'primary', 'election', 'produced', '``', 'no', 'evidence', \"''\", 'that', 'any', 'irregularities', 'took', 'place', '.']\n"
     ]
    }
   ],
   "source": [
    "print(sents[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2id,id2w = dict(),dict()\n",
    "freq,idx = dict(),1\n",
    "w2id[None]=0\n",
    "id2w[0]=None\n",
    "freq[0]=0\n",
    "for i,sent in enumerate(sents):\n",
    "    prev = 0\n",
    "    for word in sent:\n",
    "        word = word.lower()\n",
    "        if word not in w2id:\n",
    "            w2id[word]=idx\n",
    "            id2w[idx]=word\n",
    "            idx+=1\n",
    "        wid = w2id[word]\n",
    "        freq[(prev,wid)]=freq.get((prev,wid),0)+1\n",
    "        freq[prev]=freq.get(prev,0)+1\n",
    "        prev = wid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13113 73927\n"
     ]
    }
   ],
   "source": [
    "print(len(w2id),len(freq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "p,V = dict(),len(w2id)\n",
    "for key in freq.keys():\n",
    "    if type(key)==int:\n",
    "        continue\n",
    "    u,v = key\n",
    "    p[(u,v)] = np.log((freq[(u,v)]+1)/(freq[u]+V))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bigrams Language Models\n",
    "- To avoid numerical underflows and normalize the length of the sentence,\n",
    "$$\\begin{align*}\n",
    "p(\\{w_i\\}^T_{i=1})&=p(w_1)\\prod^T_{t=2}p(w_t|w_{t-1})\\\\\n",
    "\\log p(\\{w_i\\}^T_{i=1})&=\\log p(w_1)+\\sum^T_{t=2}\\log p(w_t|w_{t-1})\\\\\n",
    "\\frac{1}{T}\\log p(\\{w_i\\}^T_{i=1})&=\\frac{1}{T}\\bigg(\\log p(w_1)+\\sum^T_{t=2}\\log p(w_t|w_{t-1})\\bigg)\n",
    "\\end{align*}$$\n",
    "- Brown Corpus Data: https://www.nltk.org/book/ch02.html\n",
    "    - Genre includes, news, editorial, reviews, religion, hobbies, lore, belles_lettres, government, learned, fiction, mystery,science_fiction, adventure, romance and humor. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "res,resfake = [],[]\n",
    "for sent in sents:\n",
    "    prev,logprob,logprobfake = 0,0,0\n",
    "    T = len(sent)\n",
    "    for word in sent:\n",
    "        word = word.lower()\n",
    "        wid = w2id[word]\n",
    "        logprob+=p.get((prev,wid),\n",
    "                    -np.log(freq.get(prev,0)+V))\n",
    "        prev = wid\n",
    "    logprob/=T\n",
    "    res.append(logprob)\n",
    "    prev=0\n",
    "    for wid in np.random.randint(1,len(w2id),T):\n",
    "        logprobfake+=p.get((prev,wid),\n",
    "                    -np.log(freq.get(prev,0)+V))\n",
    "        prev = wid\n",
    "    logprobfake/=T\n",
    "    resfake.append(logprobfake)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'log probability')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2dZ5gUxdaA39plYclZEBEWkJwRkJxFFBRFRDGBGDFeMwYU41X0KmZEBcOnBAUERYKASBDJOacl57xkduv70T07PTM9Mz0z3RN2632eeaanQ/Xpmuo6VadOnRJSShQKhUKhMJIUawEUCoVCEX8o5aBQKBQKH5RyUCgUCoUPSjkoFAqFwgelHBQKhULhQ55YC2AHpUqVkmlpabEWQ6FQKBKKJUuWHJJSljY7liOUQ1paGosXL461GAqFQpFQCCG2+zumzEoKhUKh8EEpB4VCoVD4oJSDQqFQKHzIEWMOZly4cIFdu3Zx9uzZWIuiiFNSU1MpX748KSkpsRZFoYg7cqxy2LVrF4ULFyYtLQ0hRKzFUcQZUkoOHz7Mrl27qFSpUqzFUSjijhxrVjp79iwlS5ZUikFhihCCkiVLqp6lQuGHHKscAKUYFAFR5UOh8E+OVg4KhSJ3k5UlGbN4Jxcys2ItSsKhlINCocixjFu2m+d+Wcmw2VtjLUrCoZRDHHLx4sVYi6BQ5AiOnT4PwOGM8zGWJPFQysFhvv/+e+rVq0f9+vW566672L59Ox07dqRevXp07NiRHTt2ANC3b1+eeuop2rdvz/PPP8+gQYO466676NChA1WrVuWrr74CYNasWXTr1i07/UcffZRvv/0WgAEDBlCrVi3q1avHM888E/VnVSgUOYcc68pq5LXf1rB2zwlb06xVrgivXl874Dlr1qzhrbfeYt68eZQqVYojR47Qp08f7r77bvr06cPw4cN5/PHH+fXXXwHYuHEj06dPJzk5mUGDBrFy5Ur+/fdfTp06RcOGDenatavfex05coTx48ezfv16hBAcO3bM1udVKBS5C9VzcJCZM2fSs2dPSpUqBUCJEiWYP38+t99+OwB33XUXc+fOzT7/lltuITk5Oft39+7dyZ8/P6VKlaJ9+/YsXLjQ772KFClCamoq9913H+PGjaNAgQIOPZVCkXhIZKxFSDhyRc8hWAvfKaSUQd0ljccLFizo95jrd548ecjKcnteuPz08+TJw8KFC5kxYwajRo3i008/ZebMmZE+gkKR0Ch35fBRPQcH6dixI2PGjOHw4cOAZvpp0aIFo0aNAuDHH3+kVatWfq+fMGECZ8+e5fDhw8yaNYsmTZpQsWJF1q5dy7lz5zh+/DgzZswAICMjg+PHj3PdddcxZMgQli9f7vwDKhSKHEuu6DnEitq1a/PSSy/Rtm1bkpOTadiwIR9//DH9+vXjvffeo3Tp0owYMcLv9U2bNqVr167s2LGDgQMHUq5cOQB69epFvXr1qFq1Kg0bNgTg5MmTdO/enbNnzyKl5MMPP4zKMyoUipyJUg4O06dPH/r06eOxz8zc4/I4MlKtWjWGDRvms3/w4MEMHjzYZ3+gMQmFIjcipRprCBdlVlIoFAqFD6rnEKcMGjQo1iIoFAmPGpAOH9VzUCgUCoUPSjkoFAqFwgelHBQKRY5HjUuHjlIOCoUix6JGHMJHKYcEIi0tjUOHDkWczrfffsujjz5qg0TRZ/ny5fzxxx8hX9euXTsWL17sgEQKRc5EKYcoIKX0CHmhCEygkOXhKgdF7kRZk8JHKQeHSE9Pp2bNmjz88MM0atSInTt30r9/fxo3bkzt2rV59dVXs89NS0vj1VdfpVGjRtStW5f169cDcPjwYTp37kzDhg158MEHPSb0fPDBB9SpU4c6deowZMiQ7HvWqFGD++67jzp16nDHHXcwffp0WrZsSdWqVX0myZ08eZJKlSpx4cIFAE6cOEFaWlr2bxdbtmyhWbNmNGnShFdeeYVChQplH3vvvfdo0qQJ9erVy34m17Pff//91K5dm86dO3PmzJnstLp06cKVV15J69ats5/VO2T5woULadGiBQ0bNqRFixZs2LCB8+fP88orrzB69GgaNGjA6NGjOXXqFP369aNJkyY0bNiQCRMmAHDmzBluu+026tWrx6233pp9f4VCYY3cMc9h8gDYt8reNMvWhWvfCXjKhg0bGDFiBJ9//jkAb731FiVKlCAzM5OOHTuycuVK6tWrB0CpUqVYunQpn3/+Oe+//z5ff/01r732Gq1ateKVV15h0qRJ2bOllyxZwogRI1iwYAFSSq666iratm1L8eLF2bx5Mz///DPDhg2jSZMm/PTTT8ydO5eJEyfy9ttvZ4cHByhcuDDt2rVj0qRJ3HjjjYwaNYqbb76ZlJQUj+d44okneOKJJ+jduzdDhw7N3j9t2jQ2bdrEwoULkVJyww03MHv2bCpUqMCmTZsYOXIkX331Fb169WLs2LHceeedPPDAAwwdOpSqVauyYMECHn744ewZ48aQ5SdOnGD27NnkyZOH6dOn8+KLLzJ27Fhef/11Fi9ezKeffgrAiy++SIcOHRg+fDjHjh2jadOmdOrUiS+//JICBQqwcuVKVq5cSaNGjSL8wxWJiBpzCJ/coRxiRMWKFWnWrFn27zFjxjBs2DAuXrzI3r17Wbt2bbZy6NGjBwBXXnkl48aNA2D27NnZ2127dqV48eIAzJ07l5tuuik7imuPHj2YM2cON9xwA5UqVaJu3bqAFtupY8eOCCGoW7cu6enpPjLed999DB48mBtvvJERI0ZkLypkZP78+dlK5fbbb89eSGjatGlMmzYtO75TRkYGmzZtokKFClSqVIkGDRpkP1N6ejoZGRn8888/3HLLLdlpnzt3LnvbGLL8+PHj9OnTh02bNiGE8OnNuJg2bRoTJ07k/fffB7QotTt27GD27Nk8/vjjANSrVy87nxUKhTVyh3II0sJ3CmMI7m3btvH++++zaNEiihcvTt++fbPDbQPky5cPgOTkZA+bu9kMz0DxYlzpACQlJWX/TkpKMrXlt2zZkvT0dP7++28yMzOpU6eO5eeTUvLCCy/w4IMPeuxPT0/3kCM5OZkzZ86QlZVFsWLF/EaMNebXwIEDad++PePHjyc9PZ127dr5lWHs2LFUr17d55iaHatQhI8ac4gSJ06coGDBghQtWpT9+/czefLkoNe0adOGH3/8EYDJkydz9OjR7P2//vorp0+f5tSpU4wfP57WrVuHLdvdd99N7969ueeee0yPN2vWjLFjxwJkhxsHuOaaaxg+fDgZGRkA7N69mwMHDvi9T5EiRahUqRI///wzoFXsK1asMD33+PHjXHbZZYBnUMLChQtz8uRJDxk++eSTbIW5bNkywDPvVq9ezcqVK/1ngEKh8EEphyhRv359GjZsSO3atenXrx8tW7YMes2rr77K7NmzadSoEdOmTaNChQoANGrUiL59+9K0aVOuuuoq7rvvvmzTTjjccccdHD16lN69e5seHzJkCB988AFNmzZl7969FC1aFIDOnTtz++2307x5c+rWrUvPnj09Km4zfvzxR7755hvq169P7dq1sweQvXnuued44YUXaNmyJZmZmdn727dvz9q1a7MHpAcOHMiFCxeoV68ederUYeDAgQD079+fjIwM6tWrx+DBg2natGk4WaNQ5FpETghp27hxY+ntw75u3Tpq1qwZI4kSi19++YUJEybwww8/mB4/ffo0+fPnRwjBqFGjGDlypN9KPdFQ5SRnM2LeNl77bS19W6Qx6IbYrAgZzwghlkgpG5sdyx1jDgq/PPbYY0yePDng3IElS5bw6KOPIqWkWLFiDB8+PIoSKhThkwPavjEjLpWDEOIx4FHgIjBJSvlcjEXKsXzyySdBz2ndurXfsQGFQpEziTvlIIRoD3QH6kkpzwkhLgk3LSml8lhR+CUnmFQV1pmxbj+1yhXh0qL5Yy1KQhCPA9L9gXeklOcApJT+3V8CkJqayuHDh1UFoDBFSsnhw4dJTU2NtSgKBzG2De/9bjE3fDovdsIkGHHXcwCqAa2FEG8BZ4FnpJSLvE8SQjwAPABke/EYKV++PLt27eLgwYMOi6tIVFJTUylfvnysxVBEkYMnzwU/SQHESDkIIaYDZU0OvYQmU3GgGdAEGCOEqCy9ugBSymHAMNC8lbwTSklJoVKlSnaLrlAoFLmCmCgHKWUnf8eEEP2BcboyWCiEyAJKAaoLoFAoFFEiHsccfgU6AAghqgF5gcgXMVAoFAqFZeJxzGE4MFwIsRo4D/TxNikpFAqFwlniTjlIKc8Dd8ZaDoVCocjNxKNZSaFQKBQxRimHBGL+lsOcv6iWG1XkbmZvPMi5i5nBTzSgLNOho5RDgrB693F6f/Uvb/+xLtaiKBQxY+WuY9w9fCFvT7L2Hqj4COGjlEOCcPT0eQA2H8iIsSQKRew4elpbEXDroVMxliTno5SDQqFQKHxQykGhsJnrPprDZ39tjrUYCgNqxCF0lHJQKGxm7d4TvDd1Q6zFyJGogeXooZSDQqHI8SidEjpKOUSJj6ZvouvHc8K+fv8JLZpkqC58CkVOItz1WaQyLIVM3M2Qzql8OH1jRNe/9tsaABalH7VDHIVCoQiI6jkoFIqEIdwxB2VWCh2lHBIENZlHoXBj1bzkOk/phtBRyiFBUGthKxRuQu1BqJ5D6CjloFDEIQdOnCVdzQL2IdRGknJ9DR81IJ0gqI5D7qLp2zMAmPKf1uw5doYONcrEWKLwOX7mAinJggJ5I69uVGUfPVTPIUFQuiF30mXIHPp9uzjWYkRE/dem0frdv2xNM9QxBzXqEDpKOSgUCsc5fOp8TO+vOhyho5SDQqFIOJR5yXmUckgQlLeSQhHBDGmlS0JGKQdFjif90CkOnDwbazEUNhD2JDg15hAySjkkCKrfED7t3p9F07dmxFoMhY1YH5DWvp3oOazdc4IbP5vH6fMX7U88DlDKIY75YtYWxi3dBShXVoUiHJw0J705aS3Ldx5j2Y5jpsf/2nCAAyc8e6xSSg6ePOecUDailEMc8+6U9Tw1ZkWsxVAo4o5EGJC+Z8Qieg6d77Fv5MKdNHlrOmv3nIiRVNZRykGhUOR4YqVKdhw57fF73pZDAGw5GP9rwSvlkDDE3q504uwFWr07k+U7zbvRoTJo4hoWbD1sS1qK3EU8jDm4CCnt+O/wZKOUQ4IQD2MOS7YfZdfRM3z4Z2RrU7j49p90bh32ry1pKRSJRDy8z8FQyiEGLNl+JCFspv5IXMkV8UiD16fx3T/pjt7DSVfWRKjow0EphygzZfVebv5iPqMW7Qzpuhxa/hzjwImz/L5yT6zFUFjg2OkLvDpxjaVzVcMkeqiorFFm+2FtgGqbCsfsKHd9s5AN+0/SrvolsRZF4YeLmVlkRau2j5Mxh0mr9joniM0o5RAjQjEr3fT5PA7EgW90IvVe9hw7A0Bm1GofRajcPHQ+K0J0bgi3DDpRCiIxJ4kEeJuUWSnKhFOg/E2yiTZ2vWBSSnZ6ufgpch+hKga7OHDyLC3fmcnWCN1JE3jY0BJBlYMQopsQQikRm5FSqyTf/H2tvT7PJ/bCpun2pecAvyzZRevB9sb3t5v3p24gbcCkqN0v49xFVu8+HrX7JSp21Md/rNzL7mNnbBsEN2vwJbLDiQsrlf5twCYhxGAhRE2nBUoElmw/ypxNB8O61tid3HHkNF/P3Ua/bxfZJRp81QF+vNm+9AwE6/Scu5hJ2oBJfDtvW8Dzlu44ap9Q/ohwjZdP/9psmyhWePCHxXT7ZC7nL2ZF9b6JSqgdcCcr6xygB0wJqhyklHcCDYEtwAghxHwhxANCiMKOSxen3PzFP9z1zcKg5w2ZvpHF6Uf4e2NgRZJlZ+k6GTsPnRNntABkwSrWaLxM8W/R9WRxuqYwbS0LEXL01HnmbjoU8nWbD2SEFR7i7IVMy+cGy6XZGw9y/MwFy22EzCzJ5FV7PZTImj3HA/bqQzUR7zjsNqUmgvurJXORlPIEMBYYBVwK3AQsFUI85qBsCc3p8xcZMn0TPYfOp89wtyI5pUdwdLoKOHjyHBv3n3T4LuERR/WfIgB9v13End8s4Mx565U2QKcP/ua6j+eEfL/XfgvuzmqlTj12+jx3D1/Igz+4l1c1K3Pfzd/O6EU7ABgxbxv9f1zKr8t3Zx/v+vFcOv7vbwt3tEa3T0LPk1hiZczhBiHEeGAmkAI0lVJeC9QHnnFYvhyHqxfhdAXZ9r2/6PzhbEfSjrSLHs3Y+okWxz+eFOdmvXGRGSWhdh45E/QcK5Kc001zWw4Gdxd/8/d1AOw7rkVPPXQy+HKmx06f55/NhwL+V2bHTpxNrNDeVlxZewIfSik9ahop5WkhRD+7BRJCjAaq6z+LAceklA3svo/TWHFVC9ed7XDGOUoWyhfwnNMhtvZyGmaxd+ZsOkjrqqUdu+fe42fo8fk/jqWvcBPymIPF/VYaE/d8u4hlO47RsEIxTZYwXuMEsCpZMivt9VYMQoh3AaSUtq+gIqW8VUrZQFcIY4Fxdt8jGgQrMJG0aNuE4OlzzYezqTlwStj3CoXxy7S1J85e8D+oevZCJmMW77KU3r7jZ3lp/CouZIY/SGtswQ2ZvinsdKwwbulu9h6PbMU5f+XmYoh58MuSXXT/bB57j5+h8ZvTw3LbDFRCdx8L3sqPFWZZGKy3G0oFv3Gf1qPK0ufQxFNvz06sKIerTfZda7cg3git6dcLGOn0vcJl2Y6jfDwjtArHXxnsNXQ+L4xbZSmNUyH0CjbsP8mZEAb6AhEsEub707SAfBnnzLvPFzOzmLXhgM/+R35cauo2+uL4Vfy4YEfYnmE5iVDrn2d+XsGKncf4fcVeDmWc46cFO2yT5Y9Ve2n5zsyY/i+B8iOcujqsCt7P+5CVJTnp5x1w8VGI9UYs8KschBD9hRCrgBpCiJWGzzZgZRRkaw3sl1Ka5qLuMbVYCLH44EHnC+k/mw/5eB3d9Pk/fBBmhFJjYZQSFqYfYeTCHSzZfjTiyTnxyhUvTeah/1vqs99fSAGX504w89ve42dIGzCJP4KEJnDa99wOD5R4aoX6exyXo8OCrUeiJ4xOuFls5+I6wf6id6eup/5r0wKes37fSdbtPcGmOHUagcBjDj8Bk4H/AgMM+09KKSMqFUKI6UBZk0MvSSkn6Nu9CdBrkFIOA4YBNG7c2LFXatfR07R6123GGf1AM6duBWhusgDp73R19D6JgNWK0vXi/7x4J9fVvRQwxPE3pmejbKGy+9gZ8iQJyhRJjcn97Xz2gnm1asOuHqkLK6bWcJ9jqyGWmVkaYSl2V+PF69qJy625k1/7kea9FK/veiDlIKWU6UKIR7wPCCFKRKIgpJSdAh0XQuQBegBXhnsPu1i4zfMxI11/YKkhFMb389MB2HU0MvvtxBV7aH1FKYoXzBtROpFi9yBb9kscJOE5ui++1QVgYkHLd2YCwSuCUB5h6Y6jTFm9jxevc2Zuqr+K2KlsDmVtZSsiOFUavNP1bsTEbykMjWA9h27AErRyYnxmCVR2UK5OwHoppbWRywRlUXrk3fKdR07z+MhlVCxZgL3Hz7IxhqEUzzk0uzfQyzZj3X6+1cMguM7LzJIcO33BEVkCEe1gai7PKKeUg4tglaFdbNxvrzlVgqkme+23tYGvk5KLAQI2uhstOUUNmOO3KpFSdtO/K0VPnGxuI44HouOBsxcy+fLvrXSqpYWkdoUCD6Tub/p8HuWK5uezOxrZLs+Gff5tp1d/8DeFU0PXWlbGCPad8PUO+t+0DZbTcFUEKcn2hw9r+c5Mpj/Vlvx5k21P2+oAsy3jIJEnYRuHM7R5CJF4sBkxKx9Dpm+yNGAcq8CB0cLvGyuECFiDSCl9RxZtQkrZ16m0Q2Hiij08NWZFrMUwZfi8bXw4faPPAuaBWLbjGMs4xmcOyPPl7C1+j206EFmLMNT1gudtMV+X2kxPDJ66gS9mbWH9G11ITbG3Et997AxbDmZQ57KiEaf1v2kb6dsijbJFtTGLF8e7PduklI6Y1LxTzMySMTeZvKYvCvSPn//YiCCcGEwwOshCXMHaLPFs3gyFQM25/wU4JoEONssSdzw+cpljaUfqOXNWd2cduzQ+LG/jlu4OflKYBHrVpqzeZ/FMc0Yu1FrgZ85n2q4c7GTo31tYuv0oYx5q7nNMSmcsHN4ltMqLf9C5VhmapJWw/2YWuZDlkOkyQAau2XOcYbO38kGvBiQn+Z535zcLmPl0WyqXLuSIbLEikFmpfTQFUYRGqKql15fzs7dPn79Igbyhm3mi3R6yoj/nmASG8yen0+YRp11Zz1409w6y8lyRtEWMjzVt7f6oKId9x89SslBeR8x9ZgRqrD360zK2HTrFtXUupV75oqb/89Idx7KVQw7pOASc59BB/+5h9omeiM7y24o9TFgeWat3/LJdPpO7gr2M0bbjGr2uar0ylTGLdiKlzP6YRcSs9cqUsOdxRMLyncfYftjtemj1ZTONq2+TTOFiVg6On7kQVmjulbvM13twev6GX68lB+715u9rOXn2As3+O4NXJvgG4vN+1Ikr9vh4FIaSHWanervUusrVQ/+3hBbvzIyruShOEkgtt9W/rzf5dHNYrqjx2MhlPDFquc9+o003GE+OXkHfEe41GTbsO0nnIYGjOR7OOM/aveFPzPlkZmTrDUxft586r06lw//+5ou/t1Bj4BSOnvIMOnb6fGbIM8CNnL2QyX3fLQ5+ohc3fjaPtu/Nyn5JrXoBhVpZTVyxJyZeTfVfm8YDP5jnSzitTit11R+r9lLn1al+y/XgKev57x/rbL9vqHw9dxunzmkNlZnr97Ng62HSBkzyG2H48ZHLPHrFRoSwlp9SSr6a438NkqQodAVW7z7Oyl3xNcDtVzlIKV/Vv+8x+dgecC8W+Ftf+EJmVkThBj78c2PQCJOTVu3lQmbsmiASLQzHtkOnGK+PF1hdp9pqy2nWhoNMX7c/TAnd94nk3fSQ1UtwJ8eU/OGKxzNrg32z+q38H/tOnCXj3EW/5frzWVv4cvZW02PRtpIYe7GuWe//bNbMh6G8MXa18E2GGQLiXV4/s7BwVLdP5nLDp/NCu5HDWAnZXVII8bEQYqkQYokQ4iMhRMloCOc0/24193jIbatxRepNFG8YX06jiSAerAFbDwXO63AqtFCDOG4+YE/IhkiUxpo9/pdEfSvEHkwg7Jh7EmnP4b2pG4KfpFPphUkczrA+GdBJrIz2jAIOAjejhe8+CIx2UqhoYVx1K23AJB76YQlfz9lKz6Hm3VSrJMKA1N7joc3K/mbuNu4eHnz1u0h49KelHq3G7J6DxevN8t1KfH67MI0G6lVxd/rA/jU2QlUoTsgQKjd+5r+VvFuPGGB8rrOuBlsIz2rNpAQ/L9nls89IqKvzmSmkwVPWW7pWSrJNuWkDJoVk3rYbK8qhhJTyDSnlNv3zJto6CzmOKWv28eakdayLYCwgUVi9O7RnfON381ml5y5m+rWVhqokf1+518PunV2xRqBsP5rhHlAP9I5vPZTBbysCx8Q5euo8oxb6NzfGqlFg9wBp2oBJvDUp8CxigCOnAyvetAH+W8GBZiAbx+LO66bXdyZrlWuovaRg/0mmlDz3izuOqFnqgWQ1Y59J2PbPZ/mfB+TNd/O3Z2+bmQG3HMywZKqKFCv+jH8JIW4Dxui/ewK+8ZUV2cTHOsDeEU9CY/0+/8pj4/6TLN95jAaXF2PQxDWMXGg+aSicbIhkPQSzFtsOC6uLAdz8hdZbXLPnBL8sMX+eJ8csZ9aGgzSsUJzqZd1LqGdlSZICGKZDWxs5cKZdyMzi8788KxonVrv7as42CgSZ2T1u6W4+6BV4Ha5I4oYJEfo6FqFixYTs/c+aBRw8cNJdbs87LPOtX/7LoYxz3NW8IkVSUxy7TyBX1pNCiBPAg2hxls7rn1HAk45JlAOYuib8Qdh4ocsQ/+vdHjh5LtsssHynf9txOJiNH1v2VhKu890Ye4FWKtGhf2/hUIZ5i9gVGM5YoczfcpjKL/7Bku1HTa+R0v/6Fkas9jp+WbKLD6d7uhdHuy1yNEiPwS72nzjHuGWebuaBnjUzS/Lhnxs5duZ89vUz1/uuHxKIdyav93HMsLLc6OAp1scVIuWczdFw/RHIW6mwlLKI/p0kpcyjf5KklEWiIp0iqtjdAg3LLdO4zoWfdCYs32260MwyPeKtMfKtWdo1B05hyPTw528Y5Zmty+HPueGLWVssqTbP9T38/w9mvZBo91NDMZEYGTJ9Y8hzMowehQ98v9jjWY8YXK9/XLCdWRsO8NGMTbxqmB/x59rEb6jFCkvTD4UQxYUQTYUQbVwfpwWLBtGOohlNRBSqDCmdiLXjK/dFL5ffJ0Yt565vfAfH9504a2rvzU5ZT+bMhcywlgwNVK+dPn+RkyYLyE9Zsy+kWDtLth+l0gt/hCRDoAo3nuL8DJm+iaU7jnLw5LmwejvT1u73eNbHRrrDu700fnV2ML5QzHgK/1hxZb0PmA1MBV7Tvwc5K5YiEQr4Z39tDqiCwplAd+KMoYLVE7/zmwXWrz9r76S2fzYfYsrqvfy0YEf2QGm3T+b62MI/+2uL34mJoVTPczf7hgMxYj6jN3E4cz6LJm9ND/t647N6e6LFxVCfTRidPNIGTGLT/pNIKVm6w22+dPp5rQxIPwE0Af6VUrYXQtRAUxIJz7YgPueh8vPindzS+HJb0nrbRl/vULE6OezPdYHtuWvCWJpx1W73GEY4Zq6nA0TRDedduv1rc8V0PjOLPBbj/lgJL529cl048xzCrCQOnDjLPd8uyl7ZzR+nz2f6Xatj1oYDtKt+id9rL3oFyvtqjvlEu3Dw7hTlIN3gMyHu740HWbv3hGk0B6ewUrrPSinPAggh8kkp1wPVnRUrOoxeHDg0b6g8+4t9S2svTjcf4HQSKWH/ibNMDOLS6SJQSORwJ/IYPUGMLoTnL2Yxbc0+jgUZDDUqF2+cjkHkj6Zvzwh6jku0YDKauhSH+Vg/LdzBmj0nWOhn0SmXKNd+NJtGb/xpes770wIPxHovrOO9DnskrA+whkgsSBswiUMOTmDbEuXJqtEWk00AACAASURBVFaUwy4hRDHgV+BPIcQEwFrtoQibSOIuQfhOrFsteGa4CDQf5CZ9lbJIOHPerSi+/WcbD/ywhIf+b0nE6cYzofrUA/y2MvTXcfexM9kL5/jDpaj9eW9B8HE7f8ECnSCWE8ZcjJjnP0ZTJMSiXRPUrCSlvEnfHCSE+AsoCkxxVKookZMHpMPhlB+Xyxb/NW/5nruY5VeJhbIIkRXmbta8gf7dGvnSqvGIy2wzYl56yNf+uXY/dzarGNI1rjWtIyXa492BKslYBFG0g3N+QrEbMTOxzt10iGIFUmh5RSknxLLsrdRICPE4UA/YJaWMXkyCBGOMzaaqaNJz6Hx2HPHtOeyJYGJaJBgnE/pzFQ0FO1tf70/VXGHNXGqjjUQLcx4rs5nCTThxmKq/HF5b+5GflnKHnzExO7DirfQK8B1QEigFjBBCvOyYRAnOczaOO0RCuK6sWw9ZNys5TYbBNTTUyJhmZEnJfAvLS1phuG4+CDUMiRPM3niQGz+bx/eGsAvRQvW9PYk0lH4gzGZmA4xzaDVIKz2H3kATKeWrehjvZsAdjkijiD1x1Pg09ljsiKm/6UAGvb/6N+J0XLR61x7TjF04Ze9OOOJoboddSOnfbOZUVGUryiEdSDX8zgeEN0VSEffEkW7wIB5f90jiBjlB+uHT7PQa67HiRhsJK3Yd97lnzMmB5rX/W7A96jovUGylT4QQHwPngDVCiG+FECOA1UDOWgBAkU282q1PnfftUn8QxI0yEZBScvyMfQOpC7yWzJy2Zp9tafuj7Xt/cSjjHD/MT3f8XlYIw+Er7tl55AxjFjtjPvJHIG8l1zqGS4Dxhv2zHJNGYRvhjjkEWi4x3vjYQftutPhqzlbe/sNarH8rPPPzCnpeWT77dzQqyiwJfUcsZPXuEzSv4oznTCgEmuuSE/nx3+0836WG7en6VQ5Syu9c20KIvEA1/ecGKWVi+owpFHHG9CCzzMPBGAU2Wj1B18C894xohfOcMInpZQdB5zkIIdqheSulo5l+LxdC9JFSxn45qQhxIg6+QhEKqSmB10wIh9d/c0clzYkmFkV0sBJb6X9AZynlBgAhRDVgJHClk4JFg5w8CS7nPlnO4eDJc+RNtv+fMtqmM6OsHTbtV8OROQUr3kopLsUAIKXcCDi3/JBCkUto8tZ08iRZC94XLqGqhkhXXnvMYtBGRfxjpeewWAjxDfCD/vsOtEFqhUIRIZsPOtvSDnXM4YM/w18ESZGzsKIc+gOPAI+jWStmA587KZRCkVvY7HCkzVAjl26Is0initgRUDkIIZKBb6SUdwIfREckhR1EYyU4Rc5jzqbAiw0pcg8BDZ5SykygtO7KqlAocjjnHZ5RrUgcrJiV0oF5QoiJQHZUNillwvcksuJ0NrBCoVDEGivKYY/+SQIKOytOdMnJPuDKrKRQKCLBymI/rwEIIYpoP2WOGbEqVUhZyxQKhcIMK+s5NBZCrAJWAquEECuEEAk/AU6hUCgU/rFiVhoOPCylnAMghGgFjEBbFS6hUUMOCoVCYY6V6ZknXYoBQEo5F3DMtCSEaCCE+FcIsVwIsVgI0dSpeykUCoXCHCs9h4VCiC/R4ilJ4FZglhCiEYCUcqnNMg0GXpNSThZCXKf/bmfzPQAVeE+hUCj8YUU5NNC/X/Xa3wJNWXSwVSItzSL6dlE0TymFQqFQRBEr3krtoyGIgf8AU4UQ76OZvVqYnSSEeAB4AKBChQrRky5BUK6sCoUiEqz0HGxHCDEdKGty6CWgI/CklHKsEKIX8A3QyftEKeUwYBhA48aNw6oJi+ZXwWUVCoXCjJgoBymlT2XvQgjxPfCE/vNn4Gun5GhRpRR/rHJ+jV2FQqFINJwNJh8ee4C2+nYHYFMMZVEoFIpciZVlQnuY7D4OrJJS2r8ALtwPfCSEyAOcRR9XcAKRg5dLU2MOCoUiEqyYle4FmgN/6b/bAf8C1YQQr0spf/B3YTjo8yjUDGyFQqGIIVaUQxZQU0q5H0AIUQb4ArgKbeEfW5WDQqFQKGKPlTGHNJdi0DkAVJNSHgEuOCNWdChXLH/M7n3FJYUcTT8HW8wUCkUUsKIc5gghfhdC9BFC9AEmArOFEAWBY86K5yztq18Ss3s/07l6zO6tUCgUwbBiVnoE6AG0QmuQfgeMldrK5dGeIJdjyMmD4QqFIvGxMkNaCiHmAufRQlss1BWDIgKUblAoFPGMlfUcegELgZ5AL2CBEKKn04LldITDXQflyqpQKCLBypjDS0ATKWUfKeXdQFNgoLNiKRSKnMIDbSrHWgRFGFhRDklek90OW7xOEYAkZVdS5BL6t60SaxHigrubV3Qk3QJ5kx1J18qA9BQhxFS09RxAW8/hD0ekyUU4PSCtzEoKRWypf3kxVux0O3QmOfTSF8jrTIi8oD0AKeWzaNFP6wH1gWFSyucdkUahUORq/ni8dfb2VZVKxFCSyPFWBXkSzFxgyTwkpRwrpXxKSvmklHK800LlBpwekE5kShTMG2sRFDaSPwSzR61yRfjp/qu4qlIJutQxi+qvcWvjy+0QzRH6tkgz3e/cK++MlcCvchBCnBRCnDD5nBRCnHBEGoUCNR6T00hNCc0m3qJKKUY/2JzkAAVh0A21IxXLMWqULQxAWskCUblflkMWZL/KQUpZWEpZxORTWEpZxN91Cms4Xf/lhPr1mc7VbE2vweXFbE1PERsuK5Y/pN5IIArls99ef2PDyxj9QDNuvrK87WkD3OJQut4oryMDnWuVcTT9l66rmb2tzErmvHljneztgja/uHnz5MziPuGRlrEWwRFqlzNvg75yfa0oSxI6V1Uu6Vja791S3+O3U3OSc+bbEiYpFiqPYXeFH038/jaVqVyqIBB+y/7hdva4BfZuar/Ntmu9SyNOQ+uSO6c4c6KCqJ9DekTX1PZsnF1ZsQSLX/ZcNLJUoXxcU9v/WIQdRPqOudp93nW2nQ3CamUK8dTVWs+65qXOGHJy3psSAYnQlr+urrUKOLgrq/1P+2nvhhGn0TitBK4BNqsSdm9Qjsc7XBH0vCQBdzdzxtfcKSqUiI7dOh54tH1Vn32lCuULKY0Pb63Pfzr5phMKgcY6rBCN4ELTnmzL4x2rMubB5gyNoMEaCKUcQiTS/71I/hQg9m5tTtzfblOZ1fQ+6NWAogX8ezi5zBMPtqnCExFWHJFwZcXifo/9+WQb0/1CwF/PtDM9NvPptix4saMdoiUMxiLRqEIxihdI8Th+U8PI7fGRVu4pyVq16p2ME29800olKJKaEvzEMFDKwUA0xgG+vOtKXr2+FhV181KscGpWpZHLwl4vw/r/MPTOK0lOEgHtrgX1SUIF8iZH3CqMhDJF/LeCq5Yp7PdYJT9lpXLpQpQpkhqRTNOfahv8pChh5fUz/s3jHm7Jslc62y5HlkXtMOnxVvzyUHOf/a4yViy/M5V2tFDKIQArXulMsQL2/sFliqRyT8tKMTdhxff8aU06KwoskC+8ndzTMi2k8828YEQY/7rT5cTpRadCwS5zjKvlHi5WXUNrlyuqm0HNSfSxIKUcDHi/iEULpLD4pU5MfqK16fnRIPyX197q/92b64Z8TaQdsTbVSnN5Cf+9j+/6NfXZF8w1MZwKGqBmWRsG/cK4daMApihvAuVVIlC2aPBekJUydW+rShHJEc8rEjzSPnpxqpRyCEKe5CRHvAGsVpyx7mGY0cXgLdLyCudc9oSAu5ul+T3etlrp7G3X+3xrk/idORtKbJ1WV5Ri8hOtefsmTSm/7+W+6M3GN69l5tPtIhEvpqx/owulCwcffLZSb3tPuitnonTyB5iYZ9WsZIWfH2pOxxraipN2pPrsNTVsSMUaSjlYpHfTCjza/grbur7F8iduiAijd0TpEL1JrODK41Ba+Y3TtBZ2m2qlPUyBdpsFI6VTTd+5NDc1vMzj99zn2/N1n8bUvLRIdkV3Y4NyAdPNmycpYnNKLAl1FrWRHl75582wuxv77AvkBWbnjOMmaSVokqAxohK3NDlAoIbdf3vU5Zlr7Fv32a4Znv4IVq1Go+sczXl+DSsUZ8ObXWhbrbRHC71EAC+mWFC9bHAzYfniBXwqS6vOEgvD8F6a9HgrBnbznFh2+1UVQk4nGphlg2tSmNHGP83g/VUugGNE59q+yrpd9dImZ/rnyU72zuSPF5RyMBCPJpxEm0g965l22S6b4dr3jc8sTTrjw/s25p8BHXz258ujVag/3X8VHfSuvJFYW5IFmi+/txeXnX/xJWF4L9UuV5R7W1Xis9sbAXBtnbLsPHLa7/nfm4z1hIqdLrjJSYIJj7T0kKtamcI83rEqP9zb1DSQo6tc3d+6Mr8/1grQehNb3r6O1lVDUw5PdKrKhje7WDp3zWvXhJQ2wOxn28fEcUAphzghLxdIT72dJ/P8HGtRIiKtVEGq6gU5XMUWrFNTv3yxgK3BGmWL8FLXmqbHYq1s8+dNZu7z7UO+LhpiX1pMUyzVyxYO6PJrxxhcpC643tS/vBhFXa6j50/DoKI8lffXoBV9khAUTtWcGCQybFdnqw0hV/nrnjSX9NTbKS8OBr2mQskCTH6iNetet6aA7EIphzihAGcB6JM8LSr3i4ZDRqQVmr+KPNz5KIII3BxDvGWg08ORPxpKrVGF4ox7uAWPdajqqDKa8bTDcyvO6UGjF31t6fRwe7jh4LpX9+R/AKgqdlm6LiU5yXFTtDdKORiw9tI6U6tKiwXUaqVu90pw/l4gszxzuSRa8T4JTxbr53iaqDQTRPo7XUO/aYjZmWTSAo2kgrdzgmagwdhGFYqTnCQC3i9SUaqUDs1EEvr9gl9gfI/MTJdOEeueaygo5WDgpa41ublRdMLh+iOSSv3R9sHjC7koEuLsze4Nzb1lzBTAQ22r8NntjbjaQpTbzrXK+IwfGHMg3B5OpVIFeaBNZb7u0yS8BMLAOKg7+sFm9G9Xhftbu33u/cYJisMKI5h15WU/ZrtwGHl/s4DHW1Rxu0uHVB78nFy+uNskaVdl7UrHzCwV7i2evaY6ywZeHb5QEaKUg4FShfIxuGe9mNzbVYwjKauPd7QeN+ihtlVo7iessFmoZNdgrzdPm6y5kJqSTNd6l1L1Ev8hIVzkz5vsd/zAX15YeaGFELx4XU2/oSdCJsTJVzXKFuH5LjV48upq/KdTVd7pUZdnbfR2iwQrLeVgPZX7Wle2dK9ALridampOA82rODdXxps7rqrA2P4tPPaFY1ZKTbFedV5fvxwlC+ald9MK2WU3WCOw1qVFeKT9FRSP4aqISjnEDeYFtEhqCjOfbhtyJWcsfGbrVOTNk8Twvk2oX76ox/6CeZND6oH4UxoA7U08hoz0aHiZxxoXLmx3s7WjdRimSAXy5uE/napxW9MKfn35nVp4PhKsSGSlIeW99oCRL+9qbNnLx0WkWfXWTXUjHgwfcG0NJjzSyvL55YrlZ8nAq03f4ccsRBOOFUo5hEi0Z9Z/fkcjKpculB1nKNjLkSWtvz3+Wu3X1r2U+yIMQeDNr4+09JH9g1sbBHS99Nd6DbmlF+J/ZuwBtKlWOiRlGQ4vmijIWGNUWP7WazYbO2hTzdM7yOUAUN0ksGBykgjYuAibbNn9//FmR6y82zc2KMdDbatQvWzwXrGpaHrZdY0x+ouOHA/tBaUcvIiD/8QD78ozWsrJ7ts0uLxYWPGJYjE34eWuNSmiuzd+cltDbfKjgwXD5YffJM16HCWnMVZO3hVVII8vf9lU+7Lw3V9DN/tYPz/a73s8VPpWsX8BVUWEmFeHlsIZ+7k2UAXrT9nEMvZY0Fs7V1f4lyEK+fHDvVeRce6i8zeygJmp67ku1alRtrB7PkGUCNubKMRCbO0di4wE0g1KOYSKU77GVl1ZreKd2oBra1C2SCr/Gb3cY3+R/J5FwOV9VCjVvb9BBKGH/3yyDafPZwJQslDgwbWmXuGPBbFRUh73jOLbnJqSHFGMITu5XHd37dHwMtrXuIRRi3bStlppapcrGvC6uGgZmwjx6yMts3uDYD6uFaisFc6Xh5PnLgZdoyHY45uZSkfc04R7Rizy2FfSgZhloaKUQ4i0rVaa17vXpmj+FBalHyEzC0Yu3GFb+lberZRkQfcGl/HLkl0kieCBwqTUvJMAH+XwWsVV1NyzmYIt7mXi8j28d4s2yPhwuyoUypeMQNA7SJydpmklWJh+xPSYcRGbj25ryJTV+3hx/Cqf8xa82NFyizRQRM1wqVG2MBVLFqBd9Uu0OQohLuMV6tKU3/dryt3DF4YmpM6Ht9anaaXwPXykhD8eb82+E2eCnlvlkkJcU7ssG97s4sz4gAWMZiXvBkRg3H+ivwaOEOYKbdqTbej84WwAXulWi56NyzN+6W56+Rl/cSURrAybFaP21S9h2pNt2H74NEkC5m4+xOMdYrdioQulHHSshnwQQnB38zQAuje4jK/nbAWgX8tK3NMyjdaD/3JSTAA2vXUdAE90rEq+PEk0fXtGkCv8a4/8kx7hHoDGgzwKfmpKMg+0sRY7/qf7r+KihVCWJQrm5farKlCrXBGf2D1WPUgWvtSRvHkiHyr76LYGPDHKrSin/Mf/Mp1W+E+Iwde8B26tsODFjmScuxjyJDIzapUrQi0Tl2UXLlOO6/mtKganOw7/6xU4dLldUlQzNGr66Q4KfVqk+T0/T3ISb9xYh9ZXlAosmR/RqpUpnH3PjiaRe2NB3A1ICyHqCyHmCyFWCSF+E0LYv5iCgbv0BeeDxcsPhkSSz1BpuSba9G9XhU96N6R308Ct79dvqA1oLqjli+ena91LPVwF3+lRj9ZVS3l4SVxeogClC+fL9rnOm5yUXfhe6VaDoXdqgdQGXOv2hvnw1vr0DVDIA/H2TXXp305TGA+1rcJ/e2hrDeRJTgrJHNLg8mJcX9+///vntzeiTbXSpq2w5DDsFq/dUJtmlUt4tB471ixDiyolqV++KN/e4ztR7t2e9UgrWSB7idEudcpSrYxnpVxHH2RtVMGd7l3NKvJx74aW5BrYrRZPhDA3pUyRVEuKYdD12mS8u5tX9JjU6Yo2asUF9d6WlWiSVtyvpxJoz9+6aqnsd8iFnRPkAAbdoD3PK91qhWZ2C2AnMh4pVyw/nWqW8fnfhvdtHDQcuJG7mlUkzYLL+fX1ywWdZBgPiHhb9UgIsQh4Rkr5txCiH1BJSjkw0DWNGzeWixcvtuX+UkoqvfAHgKUwC9/M3cYbv6/lnpZpPNS2Cle9PYPShfOx6KVOQa9NGzDJfZ8zx+DdimTIVE48mR4wsFxAXisOMgue2QyFLLROB+k25EHHw7ufw3w+azODp2zI/r3k5U5xYY9NJDzKmUPMWLefe79bTPvqpRlxT1Ofez41Zjnjlu52XA4ATh2G9ypD/hLw/DbTUzr8bxZbD55i+lNtuMLCZE27WfHu1dQ/s5AVbb6kfofbon5/F0KIJVJK3wUviMOeA1AdmK1v/wncHENZLGOXjrU7JpJCEU3sjAEVOepdioR4VA6rgRv07VsA036tEOIBIcRiIcTigweDh721SqiF2+5XIZ5eLYUi5xPbNy6aEWFDJSbKQQgxXQix2uTTHegHPCKEWAIUBs6bpSGlHCalbCylbFy6dOiDe04S3t9tdysnZ7Sa4szqqUgkEqDwRDMibKjExFtJShnMIN8ZQAhRDXDYQBkZro6GlNKWsqjMSgnMqcOQkh/y+g+JnVOJr3rYgjAxllfEV4aZEndmJSHEJfp3EvAyMDQWcpS0GA2xVOZB1uS7h7Ss7dnhesMaMDUUlnBXo8qJFPSadGhLkLpfH4YJj0SejjfvVYavfJcvjTX9kiczMe9Ljt7D5UXkz8+/SGp0Z1UHwxXtNCVZwLkMzTHjn0+idv88egiSPEnxMenRjHic59BbCOF6c8cBI6ItwP9uqU/TStYm23TNu5QkcY67Uv4iT+Gu/LdHXdpXDxyN1MWYB5tzITPLY19KsrBnCcUEaJlY4Y5mFTmfmUWX2pfyz5ZD9oQwXv6j9t39s8jT8ubgOvvTjJBXUn5w/B4tryjJwG61uKWx5jo79T9t2HowI/v4c12qc+rcRdpWj4IJ2ELZ/+LORkxbs5+KJQvC0XRt58Jh0OIxZ2XTqVqmMGyFWpdG31PKKnGnHKSUHwEfxVKGm6+0vuCPq+vliq4YbD6DETMF5C9Ko3VyVq8jJTkpezJehZLW81YRXYQQHtFsq5ct7DEnp0DePAHDd3M0XXM9TbVzWpN/JXFJ4VTu9JqfEU1S9Pc8vry7PIk7s1LiEr9/skIR93xUH0Zca1NiQXoOBzdqZqTt8/XTc0Yv226UcogXVAFV5Hb2r/b8vWI0/PZE+On5e6W26iFuVo8NP+1cgFIO4XB0u3OVuV3p/v6kNtAWS/atgu3/BD9v+Ug4dch5eSLlzFE4bR5gUOEA4x+AJd9GkIC/d8lrMaA4Nu3EEqUcQmXfKvioHvz7hUM3sEk5bJgEs/5rT1rhMrRVcFPBsZ3w60Mw+q7oyBQJ76bBYHtXyFM4QLAGltv/3Jn7b/oTzp9yJu0oopRDqBzRY7Vsn2dzwg4UVCcKaOZFmPayfS39TH2O48k99qSnULjwV/lnK4cs8+ORcHgL/NgTJjwa+LwE6K0o5RAqVlodmRch80J46dvZmnGi8G/+U/MHn/R04PPOHLWWntCLoPdzXzwfXh5mXoDPm8PGaaFfm5O4eN43TzMOwjH71h6JXwK8QxkHDGXX5Ly9K7UKPlzOndS+D28KP404QSmHUHFVZtkFy6SAfdkG3ggc190HK0oh8wJsDrJ2g9MtEpfCyTSNauIm44C19PwtBv9mafikUUiiAXDqIBxYCxOj468el5zL0PLP26z4/hUwpG5sZIoJJu/U/E8Nh83e3dbhlTsX2Y2d8JPwYOUY+KGHTYmFhlIOIeOnS2qslA+siSD9AKVqxuvwfz1gxwLfY2eOwo+3QJZxDWInbKp2d8ld6ZkcCquV60fZ5CbO6uHXl35vb7oXz0XWqo4WgRpaZsfs7K3bbbIadz9sCbaYlzMo5RAqjg1mBUjvyDY4tEn7AJw22PvPHIXzpzWvjk1ephQnBtyyW0Y2FX470ztzDC7qS1/mZtdgp8rohEe0VrXLdJLwhLgWrBV8LAuJi1IOIePVMvVoqYeIlJrXk9E+b/ZCf9wAPjWux2EoxO+maV1h04rAZN+CYXByf5gC44BysLGl/25F+FhfzcuJ8RarbPkL3q0UQ1dih3pPLpPmxXP2phvNhpYT9z2yFbbM1H9Y7DmEeu+vr4bZ74UsWiQo5XB0O1wIvtB6Nt4DqNNe9n9uVpACsuNfmDJAm+gTSUE9vBnTF8J715GtMPlZGBOB26i/AeRwmP0+7F0RPL2zJ2DbbP/HTQlBvmM74Oe+miOBHcx4Hc4cgYPr7UkP4O/3YNUv2vaa8bDwK//n2tVz2LcKLpz13W9XpXpyn5bv5x1SosHkDOU5ZrzhnlHtzccN4YebtO1Q349DG62du2shzHzTWpo2kbuVQ1aWNmfh53vMj/87VKv8jX9eKC3dYIXeZQI5c8yw00K6ZvMXZrzuu++M14Qtl7LauSD8ijD78bPgn0+1MNXhICXMfANG3e5Ozx9j74PvrtfcZ3ct1q49uBE+ber//sb/bP8a+L+eMP4h83OH1NUqXG+zXNi47q1nVuYFmP+55kEUCpkX4NBmbfuvN2Hsvdr2z33hj2dCkMOLlWO08alAZBzU5qn89rgW2sI4r8euXtnMN7R8/6/1WGaWsFoxb58H33YL3vsfVBTmvA8junjuP33E1/EiUM/67HHNBGxk2sswXw8AuW81rPs9sCxZmYGP20juVg5Sz+hNU7XvE3s9/+wpz2tum68V0yZrndiLu9tooQAKr+zNyvKce+AqQELg8SJPH6S9kL4Ca1/7VsKfr7rXf/bHhj9g1xLYMNlwH51Zb3ueu3m69ow/9tJ6U0YunIETezyfaetfMO0lLUz12gnaQOXaifBFS70SNLHfZmW58837hczYp4VLAK3iN3JAj3S6/nf4uqMWPfOzJnBoA2ycYv7sZ45o5h2AL1poLrgrRsL3N7rPSZ/nafo5mg6DK3sOuh7aDD/dZt6C9ofrGV3ujIu+gakvwIKhWoV/cp/73BN7YflP7usWDHPLNG0gfHolTDIogq2z3NvL/g/G3g+/9NN6Vy65Xff3V0bH3e9WhAfWa0rWOPP74EYYUkfb3rlQy5cpA9yNDeN/90Et+F9Nz2fPvODuEbpkXvQ1bP3b/dv135ixb7Vn2R5xnbthc2Cd1uO0hIQ1v7oVrDdHt0H6HN/y7pGEVx5mXnTLMrgSvF/V83igAel3KsDnV/nuXzlaq3eGtoTRd/iXBeD4Lu375D7tXfVoWNpL3EVljSrZhVxofvE/6a2psnXhobme57pelvZ6XPyj2+C7G9zHMy9ohSbJoBAy9kNSsub2mZQC33WD3Uugz2+wbQ7MHuy+vwuZBXM/1LY3T4f/MyyhbawI5w2x9oxf6+sL9P9HMyu52L8Gphpi/BvvM6MQ9ByuybFtttueOui49uJ6M+Zuz98jumjPaWTvCs3FF+DpDZC3kG864x+AOj00ReFi6ffuPHW1sHb86z4uhNajWG/S4vrhRnjJa3xlq6FS+vY6yGsImTz1Be37k0bw7FYoWBL+eFqrzIy9tWM7oJgeIXbzDKjcDk7uNXmeB7XvVT9r3+dPaS62K0bCywfhyBb4vJl2rGpn7bkmP6uF/e72odtLZZHBhPR9d/e2cU0KV5ygGt3gCn0trdOHAo8PfNEK9q/Stjf9CfV6aYpqywy4qCvDo9t8r9sxH2p11xopJ3Zr+2a+BalFtUbEv4ZQ6Nd/5Bkf6f6Z7meoZXgWF6cOwT8fe+7bPg9eL+65r+GdWn6mFoN8hSGPHsp941T4qZf7vJ/7aN/3TIaSV5jnw4962T9mqVX8yAAAEUhJREFUUBJZmVoevF3O89w3SkLNG+BWkzDo8z6CK67WtjMvwJ5lWsMsfwlopvdaj+3Q3uvNf7qv27fSU8nsW6017Mx6sh/Vg4GHtHdz01RtnK3fVKjQzPzZIkDIHODV0bhxY7l48eLQLzx30n+XdtDx4C1zM+6dDt8YFrqr1MaavfzOsZ4VdCypfp1WOL2p31ur2ELloXlaq8iISHb33Lyp2DLwDPTCl5pXxnZT/TqtjKTPMT9eqrrWe/HG37O1fkZTcBdNxrhqXg/rfotM3tzMywe1nv7i4falmb+Er2nWxStH4HWTNV/yFzefAPrSfnirjH2yNXvErYST88LAg2ElI4RYIqVsbHosVyuHbbM1W7ZCoVAkMoOOh3VZIOWQy8ccEl8xKhSKXE6+MCwcFsjdyqFy2+jfs3pXuOxKzfyU00i2YQlPO3CNB5iRHMb63vFM+wCu1FZocAfc4LV2cuFLPX+XiyCcBMBz2zQTXTAKl4OnN0LvUdbTrnk9tDRZ86Hr/6ynEc94V/zV9CjH1btC8TSo1BYeX+bIrXO3cvCmml0rUQWg90/aoNzlTZy/l1M08ONRYbR73vojVOtifl4g+k3VKolAtHwCOr7qua+2If5Mj6/9X/vCztBlioSSXt4slUwaJCUqa2sXl6gC7V7QTAT36YPRFVtqv182iVV17XvQ9ll3uW37vDZYGQo3fg6NDA4FfSfB0+u1StdFV4N3UF3DYK9VCpTQBo5dDDqufXp6LQ9fojIULgPVr4X7/4KngqzH3X8+3Pp/0Ok1rbK85TtIaw2PLIQm92n3uD7YisMmXnV3/BL4klZPwX9WaeMcfb3G5kQS3D0BLqnte92lDTzzofNb/u+RUkCT/4UdMGCn1qB8eAHcPkrb3/sneGIF9JmoOU44gFIORpyeVXvX+NCvqd/b8/fVr2seKeFwucGNrkwdzSMpHK4d7N6+si88ukR7IUB7sfpNg5rdoErHwOlc/5F2be/RnjIG+x+ufh1aP6VVBi6KXe6+/vKm/q/Nk0+rUMwq6WD3DIfLvFrdfSbCZV4m3seXQec34fGl0G6Atq98Y+j+ufsZzXplLg+dm76Am4ZB+xe1wXB/VGgeXN60Vtp3qerufcUructKSn6torKDOl4B5Yyu1pc1giJBGgkut2ohtMqy9o3Q93cobZDdn+m44CXadz7dW81VHi5rDFWvDnzfTq9qvdM8eSGtJSQZnD5Ti2reaw+YuOreN13z1HPR4lHNc/EhE+eLcg0NaRbRGpSX1Agsl80o5WCkoFck1dbPwO0/w3Xvay6P1w6GuydqXdYSlUNPv0qH0K+5aah2fxeV2sBtP7p/177JWjrNH4V7Da5x906DWje5FYZZ5df8UWj1pNbNv/kb9/58BjfUq1+HUldoLwRoL1YFPc0r+2h52H++1rp7aj10GKgpj5uGaYql1BVQ3dDDEMLt6dPuRc8eAWheXS6M/1ebZ6H109Dndy2NGwzRNy/3cvOreT009JolXrqGZl7xzodKbTQFVrC0b/4YqdsL2g7w3d/+Rd/73/2rppyD0fAOKKTfVwio3F7brtJR+9/r6xMI8xeH+rdq20kBXumbhrq3+02DHgFmWXsr6Jrdtf/y6teh1w/Q8ZXQTDfhRgvu5uWybbynUQn4w1WWGvdz73t4ARTVvRRvGqqVnfK6wq52jfZ9v6Fyb/s8PLMZStf0fA9cPDRXe0e7vOs2F3sr6dI1IDlFc203UqkNlK0D982ES+trZbjateb3iTK5e56DkVLVfWcfdhzo3m56v3u7clut2wpud9eew7WW2QeGCUF20fR+/zNiG/XRZpkGompnuMarC5u3oPZtVBgHN8Byg+LxvsY1Q9dIaoDBsDz53HlYppb23UZ/jgomk4FcuP6HJvdqAQXXjHMfu7SB+TX5CmsVlgtjK9n7hTTjEUOk2z8N6fT6Xqt8d/7re42Rm/WK9u93PPenFoV7p3q6RecrDP3nhe4qffevoZ3vj2IVtPwP9B8YlYNIguQ87v8yfzGtEtu5yB55AtH4Hlj7q3vyX7E07btKB2sKx7UmSFKKe98lNSBL31/kMqjRVQuPAWSbmYw9vvYvat+P+CkDl9TUPkb8lTnvibEuyl8JD4YaIsZZVM/B1Rrr8l/N7mu0CYZCoO68Fdq9APmKeO7rNMj3vFLVPH+XN/FCK1lVa3W7WlntXnAfu/p1TVmY0dywelW3D32Pt33e3ZKveT20ec48nUi54RPtOYP9F5fqs8gLmfiPu8xMxdPcSqODQdkbe3H+BnVL13D/J1Wv0UwRd5gsSp8n1TPdawwT5lzXt3/Z044PWnlrcKf5ve2gw0BPE1Dhctq4xnUWWvxX9nFv+6uEL6mh/UeBzEyuXm8knoGuiadX9dfG6gpf6lmmA1GrOxQqqzWwytRx9yhds5xdlXh2tILwxfRACChbTxuf8NifBGXqxkXPIBiq5+DdGntmI7x5SejpmLUIbvtJC30QbAlMl4/yipFw7oR7f6snfc/xJl/hwD7Orh6Oi5ZPmHt3gNa6D5SWqwUFmt3eKWp20z4QuHUY6Nnz5PM85n1ewZLBfcONvYlCpeHZTeZpGXGNKzV/2HN/22d9z+3sUCA1o3xtnnH3UPLk1cY1rFCisjaL/XwGfmvMfIVhgD6reNN0baZxlQ6GCKV49riD4e+/vryp5zM9HUJAwyLl4Bndzt/faNv3in/l89sGHpqjhfuY+4F7nxDQf67/a+II1XPwIczCYaYcpAzN1qrmXQRH5VH0sVSGHahcncRVjryjqDq1kmICllulHLzxZxMMep1ZoQq1QCReAVIoAAuVa7yVbW95E0y5RQGlHLyxs+UQamsh3t6feMTpNbIVYZJglav0kte7J2E3CVhulXLwwc4/MVTlEMPVyxSKSLDFLBPFCtQjXL7Jb4VSDj64Ckee/CFel+zb6hDJ2qQhq4Rybm4hXkJy5EZSClg/1+X1Y/TcMuL3fzRUxtEs/657ubwMXfLZXd5cdYK/fIljlLeSN0JoXiRXdAp+LsBdv8LoO7XJXyJJcx+s3B7W/6aFAShTW4v8esu3vtfePsYdNx/gjp9h9S9QvonvClPe3PGL58JBic7N30ABkzAATe7X8qJeLy2+vfdERSe45Tv3PJCcwPUfa265gbhtpG/P9Z4/tHUyXLOIA1Glg+a22fwRLRzHtr81t1kXV7+urSfivYZD/3nawj9njsBVflbqc4LbfoTlI6GkLmPrp7X1Xa40rAp5y7ee632EQ6lqmtttg9sjSycG5O6Q3QqFQpGLUSG7FQqFQhESSjkoFAqFwgelHBQKhULhg1IOCoVCofAhJspBCHGLEGKNECJLCNHY69gLQojNQogNQohrYiGfQqFQ5HZi5cq6GugBfGncKYSoBdwG1AbKAdOFENWklJm+SSgUCoXCKWLSc5BSrpNSbjA51B0YJaU8J6XcBmwGAizrpVAoFAoniLcxh8sA4yK/u/R9CoVCoYgijpmVhBDTgbImh16SUk7wd5nJPtNZekKIB4AH9J8ZQgiznogVSgEhrsqeI1H5oKHyQUPlg0ZOz4eK/g44phyklBbjT3iwC7jc8Ls8YLpSjpRyGDAsjHt4IIRY7G+GYG5C5YOGygcNlQ8auTkf4s2sNBG4TQiRTwhRCagKLIyxTAqFQpHriJUr601CiF1Ac2CSEGIqgJRyDTAGWAtMAR5RnkoKhUIRfWLiyiqlHA+M93PsLeCtKIoTsWkqh6DyQUPlg4bKB41cmw85IiqrQqFQKOwl3sYcFAqFQhEHKOWgUCgUCh9ytXIQQnTRYzhtFkIMiLU8diOEGC6EOCCEWG3YV0II8acQYpP+XVzfL4QQH+t5sVII0chwTR/9/E1CiD6xeJZwEUJcLoT4SwixTo/n9YS+P7flQ6oQYqEQYoWeD6/p+ysJIRbozzRaCJFX359P/71ZP55mSCvh458JIZKFEMuEEL/rv3NlPgRESpkrP0AysAWoDOQFVgC1Yi2Xzc/YBmgErDbsGwwM0LcHAO/q29cBk9EmIjYDFuj7SwBb9e/i+nbxWD9bCHlwKdBI3y4MbARq5cJ8EEAhfTsFWKA/3xjgNn3/UKC/vv0wMFTfvg0YrW/X0t+VfEAl/R1KjvXzhZEfTwE/Ab/rv3NlPgT65OaeQ1Ngs5Ryq5TyPDAKLbZTjkFKORs44rW7O/Cdvv0dcKNh//dS41+gmBDiUuAa4E8p5REp5VHgT6CL89Lbg5Ryr5Ryqb59EliHFpIlt+WDlFJm6D9T9I8EOgC/6Pu988GVP78AHYUQghwQ/0wIUR7oCnyt/xbkwnwIRm5WDrk1jlMZKeVe0CpO4BJ9v7/8yDH5pJsEGqK1mnNdPuimlOXAATTltgU4JqW8qJ9ifKbs59WPHwdKkgPyARgCPAdk6b9LkjvzISC5WTlYjuOUS/CXHzkin4QQhYCxwH+klCcCnWqyL0fkg5QyU0rZAC0sTVOgptlp+neOzAchRDfggJRyiXG3yak5Oh+skJuVg+U4TjmM/bqZBP37gL7fX34kfD4JIVLQFMOPUspx+u5clw8upJTHgFloYw7FhBCuybDGZ8p+Xv14UTQTZaLnQ0vgBiFEOpopuQNaTyK35UNQcrNyWARU1b0U8qINNk2MsUzRYCLg8rTpA0ww7L9b99ZpBhzXzS1Tgc5CiOK6R09nfV9CoNuHvwHWSSk/MBzKbflQWghRTN/OD3RCG3/5C+ipn+adD6786QnMlNpIbELHP5NSviClLC+lTEN752dKKe8gl+WDJWI9Ih7LD5pnykY02+tLsZbHgecbCewFLqC1dO5Fs5fOADbp3yX0cwXwmZ4Xq4DGhnT6oQ24bQbuifVzhZgHrdC6+yuB5frnulyYD/WAZXo+rAZe0fdXRqvUNgM/A/n0/an678368cqGtF7S82cDcG2sny2CPGmH21sp1+aDv48Kn6FQKBQKH3KzWUmhUCgUflDKQaFQKBQ+KOWgUCgUCh+UclAoFAqFD0o5KBQKhcIHpRwUijAQQjQQQlwXo3uXE0L84ufYLCFE42jLpMh5KOWgUIRHA7T5ElFHSrlHStkz+JkKRfgo5aDIVQghCgohJunrGqwWQtyq779SCPG3EGKJEGKqIbTGLCHEu/paCBuFEK31GfWvA7cKIZYLIW7V0x0uhFikrxPQXb++rxBinBBiir5WwGCDLF2EEEt1WWYY5PNJx+sZ0oS+RocQIr8QYpTQ1p4YDeR3PBMVuYI8wU9RKHIUXYA9UsquAEKIonrspU+A7lLKg7rCeAttRjRAHillU92M9KqUspMQ4hW02dOP6um8jRZaoZ8epmKhEGK6fn0DtGiw54ANQohPgLPAV0AbKeU2IUQJ/dyXzNKRUp7y8zz9gdNSynpCiHrAUpvySZHLUcpBkdtYBbwvhHgXLXTCHCFEHaAO8KcWiolktLAjLlzB+pYAaX7S7YwW0O0Z/XcqUEHfniGlPA4ghFgLVERbMGi21NYCQEp5JEg66/zctw3wsZ7GSiHEysCPr1BYQykHRa5CSrlRCHEl2njBf4UQ04DxwBopZXM/l53TvzPx/84I4GYp5QaPnUJcZbjemIbAPMSzaTpBUDFwFLajxhwUuQohRDk0M8z/Ae+jLaO6ASgthGiun5MihKgdJKmTaMuOupgKPKZHgUUI0TDI9fOBtnpETwxmpVDTmQ3coZ9bBy3AnkIRMUo5KHIbddHs+MvR7PtvSm2Z2J7Au0KIFWiRW1sESecvoJZrQBp4A23pzZX6YPEbgS6WUh4EHgDG6fccrR8KKR3gC6CQbk56jpwWNloRM1RUVoVCoVD4oHoOCoVCofBBKQeFQqFQ+KCUg0KhUCh8UMpBoVAoFD4o5aBQKBQKH5RyUCgUCoUPSjkoFAqFwof/BwvcTlc+WOp9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(len(sents)),res,label='corpus')\n",
    "plt.plot(range(len(sents)),resfake,label='randomly generated')\n",
    "plt.legend()\n",
    "plt.xlabel('sentence id')\n",
    "plt.ylabel('log probability')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}