[E-15]transfomer_chatbot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "standing-little",
   "metadata": {},
   "source": [
    "**Step 1. 데이터 수집하기**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "korean-cassette",
   "metadata": {},
   "source": [
    "**Step 2. 데이터 전처리하기**(3~10)<br>\n",
    "영어 데이터와는 전혀 다른 데이터인 만큼 영어 데이터에 사용했던 전처리와 일부 동일한 전처리도 필요하겠지만 전체적으로는 다른 전처리를 수행해야 할 수도 있습니다.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "id": "generic-angle",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "id": "interesting-shell",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 포지셔널 인코딩 레이어\n",
    "class PositionalEncoding(tf.keras.layers.Layer):\n",
    "\n",
    "  def __init__(self, position, d_model):\n",
    "    super(PositionalEncoding, self).__init__()\n",
    "    self.pos_encoding = self.positional_encoding(position, d_model)\n",
    "\n",
    "  def get_angles(self, position, i, d_model):\n",
    "    angles = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))\n",
    "    return position * angles\n",
    "\n",
    "  def positional_encoding(self, position, d_model):\n",
    "    # 각도 배열 생성\n",
    "    angle_rads = self.get_angles(\n",
    "        position=tf.range(position, dtype=tf.float32)[:, tf.newaxis],\n",
    "        i=tf.range(d_model, dtype=tf.float32)[tf.newaxis, :],\n",
    "        d_model=d_model)\n",
    "\n",
    "    # 배열의 짝수 인덱스에는 sin 함수 적용\n",
    "    sines = tf.math.sin(angle_rads[:, 0::2])\n",
    "    # 배열의 홀수 인덱스에는 cosine 함수 적용\n",
    "    cosines = tf.math.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    # sin과 cosine이 교차되도록 재배열\n",
    "    pos_encoding = tf.stack([sines, cosines], axis=0)\n",
    "    pos_encoding = tf.transpose(pos_encoding,[1, 2, 0]) \n",
    "    pos_encoding = tf.reshape(pos_encoding, [position, d_model])\n",
    "\n",
    "    pos_encoding = pos_encoding[tf.newaxis, ...]\n",
    "    return tf.cast(pos_encoding, tf.float32)\n",
    "\n",
    "  def call(self, inputs):\n",
    "    return inputs + self.pos_encoding[:, :tf.shape(inputs)[1], :]\n",
    "\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "id": "contemporary-remainder",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABfiklEQVR4nO2dd3gc1dm372dmd6VV78W9N5oxxmAceu+EUBMCJCSkQAIJCYHkg/QE8r6BNAihBfImgVBCAgRiOqZjA+7dcpVsWb1um5nz/TGzq5UsWStbsiXr3Nd12Jkzs2fOmNXZ2d/TRCmFRqPRaIYHxv6egEaj0Wj2HXrR12g0mmGEXvQ1Go1mGKEXfY1GoxlG6EVfo9FohhF60ddoNJphxIAu+iKySUSWichiEVnk9RWIyMsiss57zR/IOWg0Gs3+QkQeFpGdIrK8h+MiIr8TkfUislREZiUdu8pbJ9eJyFX9Nad98aR/olJqplJqtrd/C/CqUmoy8Kq3r9FoNAcijwBn7Ob4mcBkr10L/BHch2Pgh8BRwBzgh/31gLw/5J3zgUe97UeBC/bDHDQajWbAUUotAOp3c8r5wF+Uy/tAnoiUA6cDLyul6pVSDcDL7P7LI2V8/THIblDASyKigD8ppe4HSpVS273jO4DS7t4oItfifvORmRE8ol1lMHP6WBav2szMaWPY+skKxh40nsVbmsjMz2VUy3YaGiOUHX4QS9dtxx/M4KAiE2VbrGnx0d5QR/GIUkaqJqo21pBuCEXTxrGxTWjYWYfpD1BUnEdNdR3KccguLGBiYZDI1grq60LYCvIy/GSNKaXdn8Om6hZKCzIoTAOrZgdtO1tosRwAMkyDzLw00ooLsYO5VH6ygoAImWkm6XlB/Pn5OOnZtERtGtqitIcsrEgYOxYFx2bWxGKctmaiLe3E2qJEow4RR2ErhQMIYAr4RCgcmYcVimCFLeyITdRxsBwS58bjrQ0g56BpRG1F1HKIWjZRy8GxldscB+XYXnO3DysLID4/YvrBMFGG6b4iOApsBUq581pTsR0RAREE79UwOvYNAxEDEcGfZqIUoBTKG8PdB+X+h3ikuFIOWdnpiAgCGCJ4l0EQDME95vVVVdYl7lq5A3T5RHbsTxxfjsQ/b95/xNvrvO+yav22VD7zABw8eXS3/SK79i1bsyXlcQEOnTam+7G76VuyOvWxZ/Ywbncs7sO47thj+zD25tTHnd553MWrNqNCdbVKqeKUB+mCkTNKYYVTOleF6lYAySff761zqTIS2Jq0v83r66l/rxnoRf9TSqlKESkBXhaR1ckHlVLK+0LYBe8f7n6AIw49SC0zj+Kdd+4ld+7XWfD2PXwnczr3PPUAhdfP55iLzuKXr/2UZ55bx/feeYcR5/2SsoOO4INrMrGb6jj+zUI+evJvXHb7t/hl9Hl+/PkHmJIV4OqnHuDzH6bzzD2PkFU2jquvPZs/3v13YuE2jr3yEp763MFsvOHz/O2vy2iKOVw4vYxjfv8dlow8iSvvfoubLj+MKyeY1N73cz74wwJer2kHYFZuOkedO5mJ115Fy8Fn8oOcGYxI8zF3XC5TLziUERddRNu0k3hzcxOPL9rK0qXV7NywltYdm7DCrXz45LW0f/ASlW8upmphJZu3NLOpPUZ91CbqKEyBXL9JUcDkqpvPp3bpBurW1NJQ0Uhla5SaiE1DzCZkO9jev27AEM586iW2NIXZVNvG5ro2quraaWuO0N4UIdweJdLSSLS9CSvUihVu453vjMEsLMPML4HMPJy0bJxgHjEzjfaYQ1vMIWQpmiMWJ132I0x/AMMXwPD5MXwBzLQgpi+Q2DZ8AXwBP6MmF2JFHayYjRWzsS0HK+bgWA627WBbDo7tYFsWjhVl7glTCfgMAj7TfTUN0nyG19e53fbDR1CO7X6GvC8vd9t9dbxXgHv//H0MAVMEQwTTcL9Uuu6LgIFwxHk3dxprdzz30t1AxyIf/0ktXoeRtEKPPfEbvY6XzKtv/qHbBd7oprPk2OtTHvfNt+/ptN/dNeIUzLsu5XEB3n7n3pTPzTvm6ymf+06XcXPnfp3Y4j+n/q3RHVYY39TzUjo1tvjP4STpekgwoPKOUqrSe90JPIOrTVV7P1/wXncO5Bw0Go2mT4gghplS6wcqgeSfhaO8vp7695oBW/RFJFNEsuPbwGnAcuBZIG6Jvgr490DNQaPRaPqOeL9Ye2/9wLPAlZ4Xz9FAkyd/zwdOE5F8z4B7mte31wykvFMKPOP9nPUBf1dK/VdEFgJPiMg1wGbgkgGcg0aj0fQN70m/f4aSx4ATgCIR2YbrkeMHUErdB7wAnAWsB9qBL3jH6kXkp8BCb6ifKKV2ZxBOmQFb9JVSFcBh3fTXASf3ZayVO6PM/e6VvDHtKOZ+47e8f+RxXHJICZe8637TPntuPjd8fRX/7+dnc+YfPyDcVMtfvnUsr595GrGnn2fpC3cwZu453Hn6BF6d+hghW3HqV+byQfoM3nzxaZRjM+O4I3nqhTW011Ux9phzufmUKTgvP8iSZ9dSE7GZlZfO9EtmY808m7/+dx2HH17OaZMKcT78O5teXsGypghRRzE66GfCpHxGHjcTJh/FypoQWT6D8Zl+Sg4poWj2Qagxh7ClOcpHWxvZuK2Z5toGwg3VWOFWAKIVK2hcu5XGjQ00bm+lJmLTajlEHVegDxhCpmlQEDBpq6ylfWcr7bUhmsIWrZZDm+2eG9fzTXFbQyhGQ3uUurYoda1RIiGLaMgiGrGIhduxoyHsSAjHiqIcGyMjGyM9EwkEcXzpqEAGypdG1FKuQdhRRG2HiOUgZvwnr4EYJoY/gOH9BDZ8AcQwMX0+RAQ7rt3bDspxDcnKUTjKfVVK4TgqoZ2bhmAahvsq4u1305KspMpxdv/5tL2xU9TzO8btXc/vC90ZdveE7vR82YvB+2laQxIBxOyfRV8pdXkvxxXQrYFEKfUw8HC/TCSJgTbkajQazdBCBKOfnvQHI3rR12g0mi70l7wzGNGLvkaj0STTj5r+YEQv+hqNRpOEIBg+//6exoAxJLJsRloaefWkMC9ta+a1s02eWVXD0R8s4Pl7HuRnP72GV477LEfmB6m9+hd88PgTzLnkYma8cw/PrKrhxnveQ9k2t11zJLV33sgLlc2cOTqH8m//mO89uZTatQspOWget513EJUfv05m8WjOOmUSR2c0suL+51nYECbXbzBr7kiKL/wcr2xs5M2FW7ls9mhGtm2k8sXXWLu8huqIRdAUZuQEGHXMODKPOokdRh7vbK6nNM3H6HF5lM2eRPohc2kIFLJ4ewsfb26gfnsLbTu3EG1rAsAMBAlv2kDj+iqatzVTE7FpttxAK3CNuFk+g1y/Z8jdUUdrdRvt9SGaYk7C4GsnRZ6aIgQMoT4cY2dzhLrWCOFQjGgoRjRiuQFSkRBW1DXiOlYMOxbFyMzByMzGCQRx/EGUL42YwjXgOgrLhvaYTdhyEkbbuBFXko24ZtyYK5g+A+XgGmwdhW07ntFWJYKzlGfEVY6Nsu2EoTZgugFY8cCsroZcQ6SToXV3gVnQvfGzJ/piE41fL5XArD1hOBtZ9wn71k9/n6Of9DUajaYLQ3VBTwW96Gs0Gk0yIv3msjkY0Yu+RqPRJCEc2E/6Q0LTHz2mnF8ccz23//5S7pp9Dd/97vEc+YOXKT/8FK6p/hf/qmjgs8/+mAt/9hoZhSN4/mtH8fh1f2VKVhob336WQ88+jysKanjut29REDA57pcX8+hGxYpX3yKQmcupZxzMCRm12NEQE46ayw3HjqPxsT/w/jvbaLUcji4IMv3zJ1JVcDAPv7uJqlWrOXFcLqEFz7DxlQ2sbY1iKxiXEWD0rDLKTzwaa+wRfFTVwuurdjIpy0/5rHJyZ84kVn4Q6xvCLNrcQNXWJlp2bifa2oBjRRHDJJCZS8ParTRtbqa+LpQIzEpOnBYPzMooCNKyvZX2uhD1UZummE3Y09uTA7MChhA0Depbo9S3RWmMB2ZFbGIRCyvUih0N4cQ8PT8enJWZjQpkovwZ4E/H8aURjgdm2Yqw5RC2HNpjdqdEa2KYGElBWYYvgGEIpmlgmkYiMMu2HJRDQstPaPtxTd92df14ojXTEHxJGn4nXV8E0xO7+zswSxLjph6Y1ZOe3905e4sOzOpnxMD0BVJqQxH9pK/RaDTJyIH9pK8XfY1Go0lC0H76Go1GM6w4kBf9IaHp5zRUUpbu494pXwRg6VV3su71Z3jtjrP47efu5crjxvBbaxab332Ob990MVU3fo6FDWEuv/0MckZN4ZEvz+GT677LkqYw5580jrazvsWvH1tCa/Umxh19Iv/vlEls/+P/UjhpFl87dzpjKt9jyYNvs6olwuign4M/PZ3AKVfyrzU1rPhkO83b1hJc9xYb/v0eS7c0UR+1KQiYTCvLZPQJMwgcfiLrmxVvrKulsqKBEYeUUHbUDHwzjqYyYvLx9maWbKqnvrqVUMOOhI++L5hFWm4Rjeurad7WzI5w3Ee/I9Fals/V83PTfWSWZtBW3UZLU4SmmEPYUYTsjsRs8fcEDCHdkISPfiRkEQnFiEUsYuEwdtT10bc9P/24li7BbFQgiPKn4fiDRCwnoedHbUV7zKY95hCxnUSitXjitYSe7/nsG6aB4TMQQ3Ast2CKUsotmNIl0Vo84Vu89ZpozfPRNwxJ6Pm9+ejH6UnP70qqvvW96f7xcQarnr+/GRRT1376Go1GM5zQ8o5Go9EMG0QEwz80PXNSQS/6Go1Gk4xOuKbRaDTDC73o72d2VLfyhR2LyDnvf2n98H6KbvwjJ375Gtq+cSlttsOsF1/k7At+xcQTLuCW8iq+/8hiPjOtkPAXf85l4ysYs+A+fvzKRo7MT+fwu37E1c+tYtO788kdM53rP3MwI1f9h2cfeJ+ZP76GKw4uYsON3+LtigZMEY6ZWsDYKy5hcSibx978hJo1H+FYUXY+9wwVC7awqT1GwBCmZAUYM28U+ceeQGPeRN5eWcOiNTU0VFZRPnscmTPnEiqYwPJNTby7rpbayhbaarYQaWlwA6F8AQIZOWQUjqTxoyaqW6I0xDqMuKZAls8gxzPkZpZmklWaSf26BuqjbgBXcnUt6GzEDZoG9W0RWtqiRMIxoiGLWMTqSLTmBWY5SQZUFXCTrCl/BhYGUdvxmmvEjdgOEct2K2clJVgzuxhxTZ+B6TNcQ6nPSARi2ZZKJF6LB2YlJ1rrZMjtEpjVKUDLC8yKV87anRE3HpgF3Rts4yQHZu2pEbe/E63tC4bAFPcJxlD4n7WHDAnvHY1Go9lXiAhipNZSHO8MEVkjIutF5JZujt8tIou9tlZEGpOO2UnHnu2P+xsST/oajUazLzHN/nkeFhETuAc4FdgGLBSRZ5VSK+PnKKW+lXT+N4DDk4YIKaVm9stkPPSTvkaj0SQj9OeT/hxgvVKqQikVBR4Hzt/N+ZcDj/XDXfTIkFj0SwuDHP4/Kxh5xMmcPF8w04K8eArc8/hKvnPP5Rz/v+9ihdv4160n8N/Tv0nAEE568ldcet8H3HVSCS9c/yhRR3HWd0/mFZnKy8+8A8Bhp87lC9MyWfrLB1hQ285Pz56B/e+7+fCfq9gRtpiVl84hX/gU7YedwwPvb2bjJ2tor6simF/G+ueWsKQpQtRRjEj3MXl6EaNPmQ3T5vHJjjZeWrGD6i2NtFZvonju4TjjD2dDQ4QPNzewYVMjTdW1hBuqscKtAAQycwnml5FdkEXj9lZ2hK1OGn3QNBKJ1nIL0skqySCzLI+msEVTzKHNdnZJtJacbC3LJ9TFE62FLKIRi1i4HTsawo6EEgFRTqwjMMrxZ6ACGTj+dCLxoCxHEbUdIl6itfhrXMM3jM7BWabPh2m6QVlucJZbQMWxPS3fC9CKB2Yl6/ng6uSmSI+FU+JBVYYXoLU7kvV86DkwK67nJ9PXoKHd/WElj7U3f4AHWqK1QRGYRTzLZr8t+iOBrUn727y+Xa8rMhYYD7yW1J0uIotE5H0RuWDP7qgzWt7RaDSaTvT+AJFEkYgsStq/Xyl1/x5e+DLgKaVU8tPJWKVUpYhMAF4TkWVKqQ17OD6gF32NRqPpjCfvpEitUmr2bo5XAqOT9kd5fd1xGXBdcodSqtJ7rRCRN3D1/r1a9IeEvKPRaDT7kn6UdxYCk0VkvIgEcBf2XbxwRGQakA+8l9SXLyJp3nYRMA9Y2fW9fWVILPqh0rGsf/N5lt11Fu/+5VGe/f2XefCoa/jMtEJenP01PnnmMS6//goy77mJ57Y188Xrj+H+1oks/vc/WX/Dl3llZxufPqKc3Bt/zS1/+Yj6iiWMmXMqv/3MoTQ+8FNeeXMLAIdH1/LRb15gYUOYsnQfs8+YQN5nvsQ/V9fy1ntbaNi0HMMXoGDSLFasqWdH2CLXb3BIQZCxJ08jY+5ZbI5l8uraGjasr6Nx61rCTTUEDj2OHeTwwbYm3ltXS90O10c/kWgt3U20lllURl5xBpUhi2bL2aUYekHApCjNR2ZJJlkjsskaWUx91KbNdnZJtGaKq+WnGwZZPre1t0WJhmJesrUoVqh1l2LocT0f8JKtBTuKpnRKtNbRQjG7I7GaL+A2v/fq6fmmaSQKqST89O3OideSk6wlt2QtP9BF2zeSfPRNST3RmnLsXhOt7Y2PfscYPfvo97eeP5QZLHo+uHMxfZJS6w2llAVcD8wHVgFPKKVWiMhPROS8pFMvAx5XSqmkvunAIhFZArwO3JHs9bOnaHlHo9FoutCfmUqVUi8AL3Tpu73L/o+6ed+7wCH9NhEPvehrNBpNEuJ5gx2o6EVfo9FoutAHQ+6QQy/6Go1G04UDedEfEobcTZt38P1ffIs3ph3F3CuupPAXX2ZTe4zj35/P9f/v/xgz9xzum23xpztf4/yxuQRvu4+f3v0igcxcHntiJYflpnPMfbdz43OrWfPai+SMmsLXLzuUKVte4/1fv8qm9hjzCoNU3P2/vLF0JwDHTS5g8pc/y0oZwcOvbmDHio+woyFyRk1h4qFlrG2NYApMyQow/sSxlJxyMs0lM3hzUz1vLa+mZuM22murUI5NqGQqS6vbeHtdDTu3NdO8fRPhplocK4rhC5CWnU9G4UjySjIZV55NrZdAzVZugFXQFHJ8BsVpJpmlGWSPyCJrZDGZI4tpinVnxHXfk+4ZgLN8Qlaaj3B7jIiXaC1uxLUjIexoGLtLtSoA5c8gJj4ilkPYq5oVjjm0xzoCs8K2Qyhqe4FYuyZaixtvTZ+BYboBWralXANuD4nWgE7z6C7RWqKalpAIzIr/JO/OqJocmLW76lbJida69vVEX4y4A2mw3FcVs/rgwz40EfceU2lDEf2kr9FoNEkI7sPJgYpe9DUajSYZObBTK+tFX6PRaLowlIvL98aQ+A3jz8jm66vu56Vtzbx2Jtz9wMfcct/nmPebj4k01fLij07lhU99AVOE0174LRf8/j1q1y7kuMvOpdVyuPD7p/Jy8HCe/cebKMfmiDOP5WsHZbHkx7/nlZ1tjA76OeoLR/L2Y8uo8hKtHXbt8YRmf5rfLqig4uPVtNVsJZhfxuiDZ/D5uWMJ2YrRQT/TDylh7JlHwSEnsWh7G88v3c72jfW0Vm/CCrdi+AKsb4jwTkUdaysaaKjc0W2itZyiXIpLszh0dN4uidZyfGYi0Vp2eRaZZXlkjSzGVzzSC8zqnGgtXjwlnmgt12+SlpNGNGS5gVlJidbsaHiXRGtx4onWwkmJ1uIBWfFEa6Go2xJ6fpdEa4bPSCRaixdS6SnRWvIcEknfugRnJYK0TCOh4/sNw9X2u/yhxgOzetLze0u0Zkj/avCDNdHa/mawTd1NuJZaG4oM+LRFxBSRT0TkeW9/vIh84BUU+IcXmqzRaDSDg7hzQAptKLIvvqtuwA0/jnMncLdSahLQAFyzD+ag0Wg0KSIYppFSG4oM6KxFZBRwNvCgty/AScBT3imPAhcM5Bw0Go2mL4h+0t8rfgPcDDjefiHQ6CUhgt0XFLjWKx6wqDQtzA9vfJof3ns5d825ls8dPZK/TPsCS/71ODfceg3yk2t4fnsLX7n9dO7YPoLF/36KCcedzxNXHs5lJ47D97U7+e6DC6mvWMKEeWdwz8WHUvPb23jh9c2YAqfMG8Xor3+bhQ0hRgf9HP3pqeRc/HUeW76Tt9/ZTMOm5ZiBIMXTZnPa0WM4c1IBBQGTmWVZjD/jENKPOZf14XReWFnNhrV1NG5ZTbipBjFMgvmlvLe1kffX1VJb1ewVQ68H3ERr6fmlZJeUU1CaycEjc5lanLVLorXiNJPiDD+ZJZlkj8ole0wpaWVl+MrG7OKjH9fyM003yVqu3ySQ4SctJ0AkFCMaCmGFWomFW71Ea9FdEq3Fcf3z3SRrEUvREtk10Vpr2KI9au820Zrp8149jT/VRGvxIu2pJFozjM4J13pKtJZMb4nW4t1d/faT2dtEa/2hxe9LPb+/fdMHm54fpz9r5A42BmzRF5FzgJ1KqY/25P1KqfuVUrOVUrOLCgv7eXYajUbTPSJ0HwzYTRuKDKTL5jzgPBE5C0gHcoDfAnki4vOe9ndXUECj0Wj2C0N1QU+FAXvSV0rdqpQapZQah5sr+jWl1Odw80Jf5J12FfDvgZqDRqPR9BUhtaf8ofrFsD+Cs74HPC4iPwM+AR7aD3PQaDSabhGBgE7DsHcopd4A3vC2K4A5fXl/7fI1XDzrcO6ZeDUBnmLyf1/irHN/yCHnXMJtwY+55b6FfO7okdRf/Uvuuub3ZJWN4+FvfYrK71zJ7Ad/w7l/W8z6N5+ncNIsfnj1EYxa+Fee/P0CqsIW547KYeatX+BdexQBQzhhVhmTrvsq77Rm8/D8j9m+7D3saIiiKUdy2OyRXDFrFEXVizk4J42Jp02k+LSzqMmbxMvLq3l32Q5qN26kvc5NtJaWXUBW6XheWVnNjs2NNG+vINxU6xonA0HSc4vILB5DfmkWU0fnccjIXCYVZPCCl2gty2eQ7zcpTjPJHpFFzii3WlbmiBJ8pWMgt2QXI27A6Ei0lus3CAZM0vPTCeanEwnFOqplxaJuorWYa8ztzpDrVspyiFi7VstqSwrMCsXsTkZc0+cmWIsnWhORRJCWaRq7JFpzrCjK7mzEhY6ka52CsnxGwnjrTwrQSk6AlWzE3ZNEa8kPcHuSaC3x3m4SrfW3ETfV6/fPeMPEiCtukr8DFZ2GQaPRaJIQDmxNXy/6Go1Gk4wMXb0+FQ5c4Uqj0Wj2APdJ30ippTSeyBkissZLPXNLN8evFpEaEVnstS8lHbtKRNZ57ar+uL8h8aRvKxj735c48+xbaP3wfibe9DyZxaN593tzuW/EHKZnp3HUi89wyG2v0V5Xxc0/+QYzP/ozv3roY3IuzeLdp54kkJnLJZ89ns/k7OTNW/7MO3UhDstN5+jvnU7NzAu5/S8fc1NJFod/63y2jp7HnU8tY+Oijwk31ZBdPpEJs6bxpWPGMcWoo/bZJ5h69EhGn3sy1oyTWLCugWc/qmR7xU5aqzdhR0P40rPIKh1H0ZhSKjbU01RVSXtdFXY0hBgmgcxcMovHkFecycgR2Rw6OpdpRZmUZbr/S5L1/NySTHJGZZMzpoTsMaWYpWMwSsZgZ5fukmgt6AVlZfkMcvwmQU/PT89Pxwq1YkdD3mv3hVOSiVjKTbhmOUl6vkPEcgunxAOz4kVUDF+go2hKPNmaKQl93zAE0yfYloNjO9iWlSic0l1gVpxOCddE8BsdOn480Zopu/4k352e79oKOida66lwSledv6/sj8Ipg13PH+z015O+iJjAPcCpuMGoC0XkWaXUyi6n/kMpdX2X9xYAPwRmAwr4yHtvw97MST/pazQaTRKGdESA99ZSYA6wXilVoZSKAo8D56c4ldOBl5VS9d5C/zJwxh7dVBJ60ddoNJouuB5ivTegKJ4uxmvXdhlqJLA1ab+n1DOfEZGlIvKUiIzu43v7xJCQdzQajWZfId1IhbuhVik1ey8v+RzwmFIqIiJfwU1EedJejtkjQ+JJv+ygCcz9ysOMPOJkTp4vVC9bwHN3X8k7x5xKVTjG1c//hNMfWc3Gt5/lqMsu4YeTW3ni2odoitn8+t5XCTVUc/i5Z3Ln6RNYfvOtvLCqlrJ0H6decShZV/8/fv7aBla99QlHfOM4OOt6fvPWJpa+tYrmbWtJzy1m1KGH8/kTJnDimCyir/2Ndf/6mMkXzsWYcy4Lt7fzr8WVbFlTS9OWlURa6jF8ATKKRpA3agwTJhZQV1lLa/UmYm1NgFs4JaNwBLmlRZSMzGHW2HxmFGcxKidAVqTeK4Tu6vmF+elkjcgie1Q+2WNKCYwci3/EOJysIiKBbCBZzxcyTdc/P9dvkJYbIN3T89PzM7HCrcRCHYnWuiucEkcMk7DtFkRvjVq0ev747TGb1ojVoefHbEJRC8MfwPT53JSzcZ98n3Ty1zdMN2VtcuGU3SVai+v9iYRrSX75XROtxf30uyuc0hOpFE7pa6K15HG6vn9fJVobCo4ng91E0I8RuZXA6KT9XVLPKKXqlFIRb/dB4IhU37snDIlFX6PRaPYV8eCsVFoKLAQme8WjArgpaZ7tfD0pT9o9j476I/OB00QkX0TygdO8vr1CyzsajUaThCD9loZBKWWJyPW4i7UJPKyUWiEiPwEWKaWeBb4pIucBFlAPXO29t15Efor7xQHwE6VU/d7OSS/6Go1Gk0QfNf1eUUq9ALzQpe/2pO1bgVt7eO/DwMP9Nhn0oq/RaDSdONDTMAwJTX9lTYxISz3L7jqLd//yKDf/5Bvk/fLLPLFsJ9/+2dnc0X4Y7/3t70w47nz++9U5vHHB13m/PsTlp4ynZvX7TD7xfP581RHU3nkj/35uHbZSnHX8GMZ/7zYeWFbPCy+soL5iCUXXfJeHF2/nhVfWU7t2IWYgSMmMozj3+PF8eloR8u4TrP3HApYuryHzpM+w3srhqSVVLFu+k/qNKwk1VCeqZeWNnsKI8fmcOL2Elqr1naplBQtHkFM2isLyLGaNzeeQ8hzG56VTYETw1W8m1wvKKs7wkzMqm9wxeeSMKyd99Gj8I8Zh55RhZxXTEHaNid1Vy8rISSM9zwvMyguSlpdNLOwGZ8UTre3OiCuG2W21rLborkbcROUsMznRWg9BWj6jU7WsZGNyd0bc5IRrydWy4gFa/ni/Z9Dtju4Cs3a5591UyzKETmnXejPiJo8Zpycj7p6uLbpa1gCii6hoNBrN8CGeT/9ARS/6Go1G0wW96Gs0Gs0wwTjAi6gMiTsLNzfy0oM38sa0o5h7xZXcXP8Uv/nTIr564VSWffp2fv3LRymYcBj//sGJrPviZ3hi2U4umJDPrIf/SNlhJ/KbrxxF6cu/5bnfvkVV2OKsyQUc/tMbebG1hD8+uZzqZQvwZ+byYm06Dz63iqolC1COTdGUI/nUp8Zx1RGjKNj0DpueeI7lb29lbWuEyuyJPLuqmncWV1G9bg1tNVsThVNyRk2lbGw+Jx1UytxR+YQaqhOFU4L5pWSXjqVoZA6HjCvgsFG5TC3KoDzDwFe3iWjFCooCJmXpPlfPH5VDzvhyMseMxF8+DpU/Aie7hIaIQ2PYThRO6dDzDTKDvk6J1oKFuaQX5mBHQjhWbLeFU+J6vhhmQsdviXYUTmkNW26ytYhFaziWSLgW1+t9frMjwVpS4ZSE1m9Ij4VTuur5cQI+A79h9Fg4xTQ6F1HpLdFa4l53UzglWc/v6f2pogundDDo9XzQmr5Go9EMJ4REXp0DEr3oazQaTRcO5FTSetHXaDSaJAR6dP89EBgSi/6o0WUY113CS9uaee1M+P7MhzlvUgEFDzzNGV9+CDEM7rntAjLvuYn7nlzF0QVBTnnql/xoicMPvnYcx+18nf986zGWNIU5pSSTT915FStGHMePH/yQTR++hhgmY448kTufXcmmD98l1tZEwYTDmDF3Mt84dgIT2tZR+fhjrHluLcubI4RsxYvr6njug61sX7uZ1h2bcKwo/sxcckZOoWxcMcfMKOFT4wqYWpiGY0UxfAHSc4vIKhtPQXk2k8fkMWtsHgeVZDEyy4+/bj3WxuW0rV/n6vkjs8kbl0vO+DJyxpXjGzEeKRqFlV1Kk2XQELbZ3hJJJFmL6/m56b5EkrWMogzSPT0/vTDX9dHfTeGUZD1fDJPWqE1r1OpItBZ2ffRbIlbCPz8UtbFitqvlJydWi2v4ST77Pi8HeVzPd3op4pIojJ6UXM0QwW9Kp8Ipydup6vmwq57fXfI1cBcBQ6RPen53D4pd9fz+9tEf7Hr+kMH7rB2oDIlFX6PRaPYVAvhTLIU4FNGLvkaj0SSh5R2NRqMZTnguwQcqetHXaDSaJOI2nAOVISFc5TVt54Hn1/HDey/nrjnXMj07jRM+fp0Tvz+f5qoN/OC2qzl1yQP86c7XGJHu59I/f42/2DP4033P8+Wiat780p3Mr25jVl46p/z0fHbO+yI3PL6YtW++gRVqpeywE7ninGmsXvAe7XVVZJdPZPLRh/LtkydzmK+G2if/zMonFrOwIURTzKE4zeTx9zazdXUljVtXYYVb8aVnkVM+kdIJI5k1o4QTJxdxcEkGwZ1rEMN0jbil4ykaWcCEsXkcNaGAw0pzGJ3tJ71xC/aWVYTWr6Zx3VYKSjPJG5tD7rhScieOxD9qEmbZeOzcctoknYaITVVLhMqWMMEkI25+wA3KyijKIKMwSHphdsKI68srwPaqZcUNqN0RN+Ka/gAtEbdiVouXZC2RaC1qJ16jURvbcnZNrBYPyPK51bJ8ScWkuwZl9ZRoLd46kqsZiSpZyYFaHZWzOu4j1SRrydtxI24n4+7efXR7/APrf6Nrf4/X/4veUFpH3cR+vbehiH7S12g0miTEe6A4UNGLvkaj0SRxoMs7etHXaDSaLgxV6SYVhsRvmO07WvjeTcdyz8SrAbhiyVPM+dk7bFv4X6741hf5pv0uf7j2/zBF+PJdF/HGlEu5/e6Xadqyiveuuol/raljSlaAc28+Gfvy/8f1Ty9j2csLCDXsoGTGPM47cyrXHTWKlu0byCwezaSj53DjGVM5sdii5V8PseKvH/BhVQs1EZtcv8GsvHQ2Lt9O46blxNqaMANBssrGUTJxAodML+G0aSUcXp5FbtNGosvfIT23mKzS8RSOLmH02DyOmVzE4eU5jMsLkNlWjdq6ivDa5TSs3UrDuhryJ+SRO76E3EkjCYyagG/EBOzcMtp9WdSFbHa0RNneEmFbQ4gcn0FBwKQgYLrJ1YqCZBQFCRZlk16YS0ZJPv78fIycwt3q+clBWaY/kAjO6hSUFbZojVi0hGMJPd+K2VgxB8Nn4PN3Trrm8xuJwipxPT/NZ3QkXOtFz4+TrOf7TCNJw+/Q8/1mR76UVPT8xNjSu55viOyRHt3fhVN6vM4QWKCG0oOz0JHAr7eW0ngiZ4jIGhFZLyK3dHP82yKyUkSWisirIjI26ZgtIou99mzX9+4J+klfo9FokunHGrkiYgL3AKcC24CFIvKsUmpl0mmfALOVUu0i8jXgV8Cl3rGQUmpmv0zGY0g86Ws0Gs2+wtX0U2spMAdYr5SqUEpFgceB85NPUEq9rpRq93bfB0b14+3sgl70NRqNJol4GoZUGlAkIouS2rVdhhsJbE3a3+b19cQ1wItJ++neuO+LyAX9cHtDQ94pyU/n7c/+kp997Ze0fng/n/rLDlbNf4rTv/Zl7p26kweO/wUNMZsbf3wm6874Ll/7yUvsXPkOE0+4gH/87gZGpPu58Otzyb3x13zl6eW899ybtFZvomjKkZx29mHcetIE0l57kGB+GROOmsvXz57GOWPTCT/9G5Y9soD31jdQFbbI8rl6/uSTx9FQsYRwUw2GL+Dp+VOYNq2IMw4q5cgR2RS1V2Etf4fa9z8ms3g2+SPLGDEmj3mTi5hVnsuEvDRywrXItpWE1y+lfvVm6tfsoGFjIxPPmEr+lNGkj52If8wUrNwRhNLyqW232NEapbI5zJaGdjbXtXOM3/XRzyhwtfyMogyChVkEi/NdPT8vDyOvBDO/uNdC6IYvgJjxbT+tUcsrltKh54eibhGVUNhK6PlWzHaTqiX55xumJPT8YMBM6PkBn5mSfz7EE6453er5/qRttyi69CkpmnLsToXQoWc9f09IVc/f6ziAAdDKD2TPlZQQ6IPHZq1Sana/XFbkCmA2cHxS91ilVKWITABeE5FlSqkNe3OdAXvSF5F0EflQRJaIyAoR+bHXP15EPvCMGv8QkcBAzUGj0Wj6Stxls58MuZXA6KT9UV5f52uKnAL8ADhPKRWJ9yulKr3XCuAN4PA9vjGPgZR3IsBJSqnDgJnAGSJyNHAncLdSahLQgPtzRqPRaAYJ4qXz7r2lwEJgsvewGwAuAzp54YjI4cCfcBf8nUn9+SKS5m0XAfOAZAPwHjFgi75yafV2/V5TwEnAU17/o8AFAzUHjUaj6Sv9+aSvlLKA64H5wCrgCaXUChH5iYic5532P0AW8GQX18zpwCIRWQK8DtzRxetnjxhQTd9zV/oImITrtrQBaPT+IWA3Rg3PIHItQHlG+kBOU6PRaBK4aRj6z66hlHoBeKFL3+1J26f08L53gUP6bSIeA+q9o5SyPR/TUbiuS9P68N77lVKzlVKzM8dP4avf/DUjjziZk+cLHz35N+ZddTX/PtnkryfdwNrWCNfddDz1V/+Sy+94naqP5jP2mHO5/xvzKAiYXPrFwym/7Xfc9J81zH96Ac3b1lIw4TBOOHs2PzxtMnnv/Y2Pf/Uk44/+FNeeM53LpuVh/edelj30Ou8ur2FrKEaWz+Cw3DSmnTCWCReeSKhhR5IRdxpTZxRz/mEjOGZ0LqXRaqxlC6h9byFVH1RQMHo0I8a5RtwjRuYyqSCdvFgDsm0lkbWfUL98Iw1rttNQ0UhNbYj8KWNIH+cace3cEUQyCqkLWexsi7K1KcSWxhCb69rZVt9OQcAkKz+djKIgmaWZZJZkEyzOJ1iYS6CwADO/BDO3ECO7AMeK7vLv3NWIa/oCGD4/hi9Aa8SiqT3WyYjbEraIJAVlWTEbx3ISlbN8ARPDlESAVnJQVsBndlTOStGICySqZvVkxPXHq2d182nuqSIXdBhx4xW0Ev8m3mv8SW5v7Jr704i7J+MPeyOuh0hqbSiyT7x3lFKNIvI6MBfIExGf97TfrVFDo9Fo9ifGXn8lD14G0nunWETyvO0gbkTaKlxt6iLvtKuAfw/UHDQajaavCPpJf08pBx71dH0D14DxvIisBB4XkZ/hhh8/NIBz0Gg0mj4zFPIZ7SkDtugrpZbSjU+p5286py9jVWzawZjPzmPZXWeRO/frzL3iSl65IIe/zb6cJU1hvnnjp2i/8bdc+LPX2PLe84yZew5/+tanmL3yccqunsnoX97PTfM388xjb9C4aTl54w7m+HPn8suzp1Oy6B98/Mu/8upH2/nK/87gqkOKsJ/7HYvvmc+7i6vZ1B4jaAqH5aZxyPFjmHzxifiPvQjDdwdZZeMonjSDyTOKuWDmSOaNyaM8VoOzfAG177xP1QcbqF5eQ9n5eRw3tZi5Y/OZXpRBodWAUbmS6NpPqFu6gbpVldSta2DnzjZ2hC2CEycTGDcNO380kcziRFDWlqYwWxpDVNS0sbm2jebGMFn56WSWZLqavqfnZ5Tkk1ZS5Or5+SUYuUU4wdxd/l13p+cb/kAnPb81HEvo+dGIhRVzcCy3WTGH9Ax/t3p+MGB20vMDptEnPV85tpdwTXrV87vq0bvT8+Mk6/mG9Kzn78lPYq3nD1GG8FN8KqT0WRaRC0VknYg0iUiziLSISPNAT06j0Wj2NdK/fvqDjlSf9H8FnKuUWjWQk9FoNJrBgJZ3oFov+BqNZrhwAK/5KS/6i0TkH8C/cNMrAKCU+udATEqj0Wj2F7pcoksO0A6cltSngH2y6PuCWaz67dm8Pu0o5n7jt7z+6Sz+coRrxL3hpuNo/9bvOf8nr7L53ecYM/ccHrrpOOas+DvPfuk+LtjwLjd6Rtz6iiUUTDiM48+dy/+cN8M14v78UV75sIqqsMXNhxbjPPc7Pvn9C7z1yY6EEXdWXjqHnjSOyZeejP+4S1ht5SaMuDMOKeWCmSM5bmweI6wanGVvUPPWu1S+u57q5TWsaYly4vSSXYy4kZUfdmvErY3aCSNuOLOYGs+Iu6khtIsRt705QmZJZqegrJ6MuF0Nubsz4pppQUxfoFcjbjxAy7aclI24AZ/RJyMukLIRN1lj1UZczd5wAK/5qS36SqkvDPRENBqNZrBwIBcaSdV7Z5SIPCMiO732tIgMaHUXjUaj2R+IVy4xlTYUSfUL7c+46UBHeO05r0+j0WgOOHRELhQrpZIX+UdE5MYBmE+3HDwqmxfHz2ZBbTuvnQkPHnEFa1ujfOe206j50p18+vaXqFz4AhOOO5//u+k4DnrvPp667i+8Uxfixecq+M8/XqVpyyoKJ83i9E8fw8/PnErBO4+w8Od/59XF1ewIW4zL8BN76n/45J6XeGtZR5K1WXnpHHLKOCZdeirmcZeyKpzJE0uqKJ18EAcfWsqnDx/Jp0bnUhbZjrXkdWre/oBt765nx8pa1rfGqI5YnDuugKlFQQqjdW6lrJUfUrt0A3Urq6hfX09NbYjKkEVDzKbVcrAKxhLJKKSm3aKy2U2ytrHerZSVrOe3t0Q66fmZ5YUdSdYKy5CcIpz0bFfTT8tO/HumoufHE651p+fHk6zF9XzbdlLW89N8Rp/0fLfCVWp6flyLT0XPh91Xyuqq58se/oX3puf398PiEF2HBhWClncA6kTkChExvXYFUDeQE9NoNJr9hYik1IYiqS76XwQuAXYA23ETpmnjrkajOfDwfgGm0oYiqXrvbAbO6/VEjUajGeIIbg2HA5XdLvoicrNS6lci8ntcv/xOKKW+OWAzS6J+2RreN8r54b2Xc9eca2m2bG69+zN8cvrNfOGWf1G9fAHTT7+If3zrU5T9+w7++r1/8nFjmBOLM/jaX56ntXoTJTPmccGFR/KT0yaR/t8/8P4vnua1VbXURGwmZgY4bu5IFv7vC7y9rp6qsEWu3+DI/CAHnTWR8ZeegzH3QpY0mTz2yVbeWlzFrFnlXOAVTSlu20Lsk9eofutDqt7fSNXqOta3RqmOWIRsxYziDPJC1bBlGaFVHyf0/Lr1DeysD7EjbCf0/KijaE8voLbNorI5wpamMBvr2hJ6fmtjmLbmCKHWCJGWZrLKc5P0/EKMvBLM/GJXzw/mooK52GlZtMdcrTxZzzf9AW/bjxkIYvgDmL6Au+0L0NgeJRS1CYWtTkVTrKiNYytsz1ffjhdR8bT85KIpwYBJwIzvu60vej7QSc/3mx36fVc93zRS1/Ohez2/v7T85PHjaD1/6DBUpZtU6E3eiadeWIRb9rBr02g0mgMKNyK3/+QdETlDRNaIyHoRuaWb42ki8g/v+AciMi7p2K1e/xoROb0/7m+3T/pKqee8zXal1JNdJnpxf0xAo9FoBhv99Zzv1RO5B7eI1DZgoYg826XA+TVAg1JqkohcBtwJXCoiM4DLgINwXeVfEZEpSqnuf7qmSKqG3FtT7NNoNJohjisXptJSYA6wXilVoZSKAo8D53c553zgUW/7KeBkcfWl84HHlVIRpdRGYD19rEXSHb1p+mcCZwEjReR3SYdyAGtvL67RaDSDjr4FXhWJyKKk/fuVUvcn7Y8EtibtbwOO6jJG4hyllCUiTUCh1/9+l/eOTHlmPdCb904Vrp5/Hp01/BbgW3t78VSJOoofP38Lv/afQICn+P4TN/DYiAu45eb/o3nbWo64+HM8c93R2L/5Ng/8z+tsaIty7qgcTrrnS1z542WMPPIsvnDxIdz8qTGE/++nLLjzRV7Z0kSr5XBwThrzThzL9K9exM8vuJOaiE1xmslRBRlMu3A6oy86HzXnAt6uaufxjzfz4eIqqtdV8JNLL2bOyGzy6tYS+egVti9YROX7W9hW0cj61ii1UZuoozAF8lu34mxcQvuKxdStqKB2ZTUNFY3saAyzI9wRlGV7pvLqdteIu6kxxOa6dipqWqmqDyWMuOH2KJGWZmLtTWTMKCSzrBB/YTzJWjFkFSaSrNn+DNqiDu0xpyMgyzAx/W4AVnKlLJ9nwI0HabWGLaJRu1sjrhW1sW03OMuxHQKeATfgM8gImJ2CspKNuAGf0cmI22G07TDiJhteHcdO2Yjb3ZNXT0bcOKkacftqdNVG3KGLKIX08rlJolYpNXsg59Pf9KbpLwGWiMjflFL6yV6j0QwLRDn9NVQlMDppf5TX190520TEB+TiBr+m8t4+s1tNX0Se8DY/EZGlSW2ZiCzd24trNBrN4EOBclJrvbMQmCwi40UkgGuYfbbLOc8CV3nbFwGvKaWU13+Z590zHpgMfLi3d9ebvHOD93rO3l5Io9Fohgxql7CkPRxGWSJyPTAfMIGHlVIrROQnwCKl1LPAQ8D/ich6oB73iwHvvCeAlbg21Ov21nMHepd3tnubtUBIKeWIyBRgGvDi3l48VcoPGs/nqg7jhT/9ltYP7+e764p46Ob7cKwoZ3zlav5x2XQqvnE5f39sBa2Ww2fnjGDu725mYemxTDr+XW7+3Ew+O0ax81c38t69b7Ogth2AeYVBjrxgKhO/fDWN00+jJvILRgf9zBmbw7TPzKT8MxfTNuV4Xqto5PFFW1m+tJqdG1bTumMTx4/NJX3rR7S9/zKVCxZT9WEVG7c1szVkUZ+k5+f6TezVH9CyfAl1yzdSt6aWhopGKluj1ETcoKyQ3aHnBwyhoj7ElqYwm2rbqKhppbohRFtzhPamCO2tEWJtTUTbm7BCrWSNLMZfVIrhFU0hMw8nPRcnPYeYmUZ71KYt5tAeUz3q+clJ1sw0V9f3BfxEIhZWNF4sxfaCsdwCKsl6vm1ZSVq+sVs9P5CccC1Jz+8akOUkaap+08AQetXzu0r6qej5vSVY21vtvbu3az1/kKNUqk/xKQ6nXgBe6NJ3e9J2GOjWBV4p9XPg5/02GVJ32VwApIvISOAl4PPAI/05EY1GoxksiHJSakORVBd9UUq1AxcC9yqlLsYNGNBoNJoDDAWOlVobgqSaT19EZC7wOdzoMXD1KY1GozmwUPSrvDPYSHXRvxE3AvcZz7gwAXh9wGbVhVV1Nkt//yfGzD2Hk+cL7//9D2SVjeNbN17ILRNaeffUs3jywyoKAiZfvHg6M351J3+vyeeO373LvdfN5VgqWHvrL3j9qdUsaQqT6zc4tiiTw744h5Ff+AoVOdN55J3NTM9OY/ahJUy9ZA75536O7blT+O+KnTz+4VY2rdxJfcVy2uuqcKwogZWvUv/Oa1S+vZLtH+1gfW07VWGLppiNrVxtPtdvMCLdT/0HH1C3fBP16xuo29xEZcgtgN4Uc7X/ZD0/y2ewtq6Nip1tbK5ro64hRHtzxPXPbwsTbaknFm7FCrViR8P4SyZiFpZh5pckiqWoYC5hfLRHHdpiDiHLoSViJRKqJfvmu/p9sLOe7yVPi0XshD++q+mrRAGVuKavHBvHiib0/GDA16lgSlzHNw3ZRdOH3vV8Zdud9PyuvvrQoecbSep2b3p+/H2Qmp6/J/m3Bto3v7traPoDBc4wX/SVUm8Cb4pIlohkKaUqgH2SYVOj0Wj2NUNVr0+FVAujHyIinwArgJUi8pGIaE1fo9EcmPSfn/6gI1V550/At5VSrwOIyAnAA8AxAzMtjUaj2U8oBamnYRhypLroZ8YXfACl1BsikjlAc9JoNJr9yoEs76S66FeIyG3A/3n7VwAVAzOlXQk1NTDvW5/npW/MJXfu1xkz9xzu//axzF39BM8cfS+v7GzjsNx0zv/OieR/526+O389Tz71CtXLF3D0WbV88ItHePm9SqrCFiPSfZxwaAmHXXsSGeddyzstmdw/fw0ffrCNf5w6jimXnYz/+EtYbRfw9EeVvLhwG5Vrq2jasopQww4A0rILqH7+WSrfW8eOJTtZ0+JWyWq13A9K0BSKAj5GBn2UFwbZ8cE66tY1sHNnWyLBWlPMrZIFbmm2uBE3x2eyorKZzbVtNDeGaW+O0N4SIdLWSqytqZMR14qE8JWOwcgtSiRYc9KyabcU7TGbNsshFHNoCls0RaxdjLgJA65XNcsXSMMwDXwBE5/fxEpKtmZ7xttOgVlW1DXkxqKuAdcLyOpqxE0008AQ2W2VrLgRV9lJwVmGsduArLgBVyQ1A27y9Xoz4u5pAaVUjLh7U51JG3AHkv4Nzhps9KUwejHwT+BpoMjr02g0mgOP4arpi0g68FVgErAMuEkpFdsXE9NoNJr9Qj+nYRhs9CbvPArEgLeAM4HpuD77Go1Gc0AiDG9Nf4ZS6hAAEXmIfkjruSeMGFXG66fFeHnaURxzw+/415ePpOFHX+Gue9+nKhzj05MLOP4P11Fx6MV89o8fsHT+m7RWbyJ3zHRe+cLdvL6jlZDtMCsvnblnTGDKly8jevTF/H1VLQ+/sYwNH2+gYfNyZvzPtdiHn81rm5t54pMNLFq8nep162iu2oAVbsXwBUjPLSJn1FTWPXcvWzc1srEt1qlgSpbPoDTN1fNLRmZTMLmAqg+rqGqOsCNs02x1LphiCgRNgyyfQb7fpCBg8EJVM61NYUItUUKtESItjYkEa3Y0jBUN4cSiOFYUKSjHDnoJ1nxB2qMOIUvRFnNoi9o0RSyawjFaozZmIH1XPT8pwZppGvj8JobPwOc3iEasHhOsxbX8eHBWMGD2mGDNNISAaeA3BMOQ3RZMgc56vnLsXvX8hC6fotCdrOfvrmBKsuSeqg7aHQeanr8XUx8iKLAPXO+d3j7LCSmnr0VURGS0iLwuIitFZIWI3OD1F4jIyyKyznvN34N5azQazcAQT8NwgGr6vS36h4lIs9dagEPj2yLS3Mt7LVwbwAzgaOA6r7r7LcCrSqnJwKvevkaj0QwaDuQsm73l09/jpGpeLv7t3naLiKzCLep7PnCCd9qjwBvA9/b0OhqNRtO/DG9Dbr8gIuOAw4EPgNKk4iw7gNIe3nMtcC3AyNws7px7HbVRi1dPV7x5zAk8vXwnpWk+vvHFmUz82a95eLOPu37+Gls+fBmAMXPP4eKzp/HcOfdQEDA5ZUw+h37haEqv+ArrghO4/+UNvPzOZqqWL6a1ehPKsdk+5XReWLyDJ97fwpbVNdRXLKW9rgrl2PjSs8gsGU3BmMmUjctj+TP1bA3FEvp8wBAKAialaT7GZAfIn5BH4dQi8qeM5q35FYkEayG7oyJPh2++Qa6n5+dmp9FY00aoNZpIsBZtb8KOhLDCbdielh/Xw+3sYlRaNiFlEkpKsNYYsmiNuv75rRGL5oiVpN93n2DN5zfxBYyEtt/WHNnFNz+u4cf1/ISm7zd30fPjSdb8hoEpbjEUvyG9JlhLbHv9fsPYpfh5sp5vSGo6c1cf/lQSrA0mLb/v1+/fax34Wn4SB/Civzef6ZQQkSxc3/4blVKdJCGvDmS3dcmUUvcrpWYrpWYXZgYHepoajUbjEk/DkEobggzooi8iftwF/29KqX963dUiUu4dLwd2DuQcNBqNpm8olBVLqe0NqTi1iMhMEXnPc4ZZKiKXJh17REQ2ishir81M5boDtuiL+zv2IWCVUuqupEPJld+vAv49UHPQaDSaPqPYV0/6qTi1tANXKqUOAs4AfiMieUnHv6uUmum1xalcdCA1/Xm4tXSXiUh8Mt8H7gCeEJFrgM3AJQM4B41Go+kTCtXJtjSA9OrUopRam7RdJSI7cVPiNO7pRQds0VdKvU3PcSQn92WsqqomivMKuO43F3PXnGvZ0Bbl3FE5nPT7L1A570uc+Y8lLJ7/Ns3b1pJdPpEZJx7D/zvvIE7OaeK+nDTmnTiW6V+9CHXClTy5po4//WsxGxZvpm79x8TamvClZ5E7agp3vL6B9z+pYsfaDTRv30CsrQkxTDIKR5BdPomScWVMmljACdNKWNUaTQRk5fqNRIK1svIsCibnUzBlBPnTx5I2fhpbQ8/1GJCV4zMoCJgUpfnIKAqSWZpJS0OISEszsfYmom1NuwRkJRskrWABbTGH9kSFLJuWqEVT2KI1atMUidEatmhqj+FPz3KDszwjbncBWXGDrukzvGpZPQdkJQy5jp2onNVTQFbcmOszjZQCspLpLSCra9Ws7uguEVtfArL6aoDdn0bc/jbgwnAz4tKXyllFIrIoaf9+pdT9Kb43JaeWOCIyBwgAG5K6fy4it+P9UlBKRXq76D7x3tFoNJqhQ5/y6dcqpWb3dFBEXgHKujn0g05XVEqJSLdOLd445bhZjq9SKuFadCvul0UAuB/3V8JPepuwXvQ1Go0mGaX22kjbMZQ6padjIlItIuVKqe27c2oRkRzgP8APlFLvJ40d/5UQEZE/A99JZU4D7rKp0Wg0QwuVkC57a3tJr04tIhIAngH+opR6qsuxuBekABcAy1O56JB40i/OS+cLq/7Dr1bYBHiKm288hlG338Vdi1t44LaXqPzoZQxfgAnHnc/nz5vOdUeNIn3Boyz+w1Nc+qOzyL/kWlbKCP74/BoWvLuF7Ss/oa1mKwBZpeMonHgIEw4q4cX/rqZx0zJCDdUox8afmUt26TjyRo2jfHw+86YWM298AQeVZLLEUQRNId9vUpbuY3RuGgWTCyiYVEj+9LFkTZqEf9x0nMKxNMU69MGgKQTNDi2/IGCSnZtGVkkmGUVBsspzaK+r7pRgrWtAVjINYTuh58eLpbR6mn5b1NXyG9tjtEYszECwU0CWL2C6mn5SQFaytp/Q9JOKpSQHZDlJH/5gkqZvehq+33STpHXo+pLQm3sLyEre95tJGn43AVnJGn9XevvD7G8tvzt2N0aqSeJSRQdk9QNx752Bp1unFhGZDXxVKfUlr+84oFBErvbed7XnqfM3ESnGtZ0uxk2D3ytDYtHXaDSafYfqiyF3z6+iVB3dOLUopRYBX/K2/wr8tYf3n7Qn19WLvkaj0SSj2Fcum/sFvehrNBpNJ/rkvTPkGBKLvjVqPLPuWs36N1+g9cP7ecWcwcV3fczaN18h2tZE8bSjmXfqIfz4zGlMrvuYjbf8Pz56Yjnv14f4zt+e5e6l23nqzQ/ZvGQlTdvWYkdDpOcWkzfuYEZPG8kph4/gnOmlHPfo37GjIcxAkIzCEeSMmkrZuHxmTini2ImFzBqRw7gcP/7qNR3J1TJ8FI7NpWhqIXlTRpE3dTz+cdOQ8olYeaNosN1/4oAhBE0h0+zQ8vMz/QSLMsgqySCzNJNgST6ZZQWEXtyRKHzek5YvhokYJo3hDr/8uJ7fEnG1/Nawu90ajtEStvBn5nb44ScKoBuejt9F3/cZWNGYe327s1++cmzs+L73RBTX9OMavt8rgu43XR3fbwimp+mn4pufvN81uVrXPthVG0/FyNZVz9+dlr+n2ntPer7W8gcx/ei9MxgZEou+RqPR7Dv0k75Go9EMH/ad985+QS/6Go1Gk4RCJeo4H4joRV+j0WiS0U/6+58Nm3bgf/05Rh15GifPF5bOv4/W6k3kjpnO7AvP40fnzmBeWjXV932X/z74Hu/sbKM+alOW7uOzDy+kYvEm6jcuIdbWhD8zl/xxBzNi2gTmzRzBuQeXMbs8k5ydK1GOnTDglowtYcrEAo6fWsJRo3KZmJ9GsGETzqIlNC9fzGG5aZSMzHYDsrzkaoFx0zBHTsHOH0WzkUFNm8W25nayfG5ytXyvOlZ+wEdmaQYZRRlklmSQUZJLZnkhweJ8/MWlRJ9e321ytThimBi+AGKYbG+N0OJVxmqJdhhwG0MxWsMx2qM2rWGLaNQmkObrFICVCM7ym5g+wTANAklBVnYk1G1ytYRB104KzvKb3SZXi1fMMkQS26kacOOYSZWtkpOrdTLs0mHMTDVSMpWArOFmwIVhbsQF15Abi+7vWQwYQ2LR12g0mn3HvgnO2l/oRV+j0Wi6ouUdjUajGSYo1R/J1AYtQ2LR96Vn8r2f3cgtx48jd+7XyS6fyJzLPs8Pzp/BKXmt1P/lx7z64Du8vaWJmohNcZrJOeXZTLtwOv/zzL8ThVIKJ82ibMpEjjysnPMOKWfuqGzy6tcRmf8yGxcsonjaGRSNKWXK5EJOmFbCnJF5TMwPkNVSifPJJ7StXkrt0vXUrqxm6rGjOxVKMUe5Wn6TmUVNyKKyuY1NjSE217UzIt23S6GUuJbvBmQV4i8swswvwcwvxgot7lXLN/1uMZTtLRE3wVooRlO7G4TVGrFoCccSWr4Vs7FiDoGgf5dCKT6/sYuWn+YzCAZ82NFQr1p+fJ5pPmO3Wn48UMvsQXfv6Y9MOXavWj50FFjp6x9rqlr+3srcWssfWmjvHY1GoxkuKIWy9aKv0Wg0wwKlFE7M2t/TGDD0oq/RaDTJKPST/v7m4NE5fHPdQ7zxlZc45obfcfu5MzguWMvOP/+IVx58l7e2t1IfdbX8c0flMP2igxl10QU4s85FnfQ9iqYcSfmU8cz1/PKPHJFFbu1qIv99hY1vLqJq4TY2b2jg2LtuSvjlj89LI7NpC84nn9C6ytXy69bUUL+unqrmCJf85jLSJs5I+OU3GhnUtFtsa25jS1OIipo2Nte1sa22nZuy03bR8hN++YVFmIVlmPklkJmPE8ztNrlaVy3f8PkxfAG2NLS7idXCFk2haCe//FjE1fNt28GK2qRn+nfrl+8WN/f2TWOXQindaflx7TPdNHr0yzfE9bWPFzhPvr/daflx4naA3Wn50PcycPHz96eWvyfjD4Ser+mMXvQ1Go1mmKCUwtH59DUajWb4cCB77+jC6BqNRpOM572TStsbRKRARF4WkXXea34P59kisthrzyb1jxeRD0RkvYj8wyui3it60ddoNJok4t47qbS95BbgVaXUZOBVb787QkqpmV47L6n/TuBupdQkoAG4JpWLDgl5p37pam7/ZgMBQ3j1dMXG31zH015lrJCtGJfh55SphUy75AhKL7yUxtFz+GdFA4//bQmHn39BojLWIcXp+Dd+QOsTr7DmzSVUfbSDiqoWtoZi1Edtbj99aqIyVuydj2hYsZy6FRupW11HQ0UjW9tj1EQsmi2H4EkXY+eNosb2UdNusbmxha1NITZ6Btwdde20NUdoa45QNrOkU2WsYEk+vvxijPwSfIVlOBl5OGnZOMFcoob7ZR2vjCWGieEPYHjG3LgB10wLYvoCbGsIJSpjtYYtNxAr6ngBWZ4h11I4lkNmTnqnyljBgEmaZ8RNNuDG+6xoKJEcrTsDbse2m3AtXhmro0pWZwOua9zdfVK0boPSekis1tWA21OSs57oyYDb3Sh9Da4aCAOuZt/h7BtD7vnACd72o8AbwPdSeaO4H96TgM8mvf9HwB97e69+0tdoNJpkPJfNFOWdIhFZlNSu7cOVSpVS273tHUBpD+ele2O/LyIXeH2FQKNSKv5zYxswMpWLDoknfY1Go9ln9C0it1YpNbungyLyClDWzaEfdL6kUiKiehhmrFKqUkQmAK+JyDKgKdUJdkUv+hqNRpOEov+8d5RSp/R0TESqRaRcKbVdRMqBnT2MUem9VojIG8DhwNNAnoj4vKf9UUBlKnMaEot+1FFcPKucmdeeyF1zrmVDW5SgKRyWm85hx45m6uUn4j/hMtapQv68YgcvPPs+W1dX0rRlFYufuJVRTh3O8ueo/fM7VL63jh1LdrK+NUpV2KLVcv/nBk1h0o73ib7xEVXL1lO7fCt16xqoq2mjMmTRELNpijlEHffLeGv6GKrrYmxqbGVTfTsVNW1sq2+nqTFMW3OYUEuUUEsLsbYmyo+aREZJPmlFBYlALCO3CCeYi5WejZOeS7ulaI86hCzL0+4DiGliJun4hj+ALxB0Nf1AEMMfYHNtG5GkpGpW0rZtOdi2g+O9pmf6CXTS8jt0/HiitUBSc2LRTrp9/A8huQ/AcWzSfV5Alqfl+w2jk46frOunmmwtjhnX7nvR8vdWd+/69v5OkqZ1/CGCUjjRfZKG4VngKuAO7/XfXU/wPHralVIRESkC5gG/8n4ZvA5cBDze0/u7Q2v6Go1Gk4wCx3FSanvJHcCpIrIOOMXbR0Rmi8iD3jnTgUUisgR4HbhDKbXSO/Y94Nsish5X438olYsOiSd9jUaj2Vco9k2WTaVUHXByN/2LgC952+8Ch/Tw/gpgTl+vqxd9jUajSUbRqY7zgcaQWPTLZ4xl/PyXuWdxFQGe4rIjypl+yWyKL/wcO4sP4ekNDTz2zBY2rFhC7YYVtNVsxbGimIEgeU/+nDVvL2P7xztYv72NqrDrk28rCBhCcZpJaZqPMRl+1v3mD9SurqNhUxOVISvhkx+yHWzPrh4whKAp/GddLRU7XZ/82oYQrY1h2lujhNuiRFvqibY3YYVasaNhCo86IlEgxcnIw0nPxU7PJmoEaIs5tLdZhGKKpkiMloiNP5iV8Mk30zwNP0nHNwPBRCGU5qZItz758URrylHYloVjRSnMCiR88oN+s5OObxrSSc/3G27CNdjVJx9cHT+Osm3SfEa3PvnJ+8mFUJLH2h1uEZVdk6p1p+PvSR6yVH3y+xoD0Ns1NIMZpdMw7Aki8rCI7BSR5Ul9KYUdazQazX6jb376Q46BNOQ+ApzRpS/VsGONRqPZLyilsKNWSm0oMmCLvlJqAVDfpft83HBhvNcLBur6Go1Gs2coT9LsvQ1F9rWmn2rYMV4487UAY8p7PE2j0Wj6F105a2DoJewYpdT9wP0AmSOnqKOufYimbWtp/fB+GkfP4eWKBh5/YytrVrxKzfqVtO3cih0NYQaCZBaPJmfUVErH5PHk969PJFSzlRvok+uPG299FI7NpWhqIXlTRvHv/329R+Ntlk/INA0KAiYFAZM/vbs5kVCtO+OtFQnhWG5wk/+gK3DSc4l5CdXaYg7tYYdQLEZL1KIpbNEUsWiNWrRELAJZ+YmEat0Zb03TwBcw8fkNWptCCeOtbbsBWY7tJIy3yrYT8yjISuuUUK1rM71kafHKV44Vc/9f9GC8TWwnB2f1YLxNTprWmwG36/F4cNbujLd78pM12cB6IBlvdWGtvUSBsntcmoY8+3rRTynsWKPRaPYXCrWvsmzuF/Z1RG487Bj6EDas0Wg0+wwFylEptaHIgD3pi8hjuLmii0RkG/BD3DDjJ0TkGmAzcMlAXV+j0Wj2BKXAjurgrD6jlLq8h0O7hB33RqixgUBLPSOPOJmT5wubV71A46ZltNdVuZp5Zi7ZIyZSMGYiZePymDulmHkTCjmkJJNf3hb2grB8jEj3MTIrQMHkfAomFVIwfSxZkycRGDcNp2gcS257MXHNoCkETYMcn0Gu36Q4zSQ7N42MwiBZpZlsWlFFrK2pk45vx6IJ/TxZl27Jn0hbTBEKObTHop00/NaIRXPEoqndK4QSsQjmlyWCsnx+E18gruN7BVD8JobPwOc32LmlqUPL964dT5SmHFfPd7ztkuy0Dv3eC8byGwZ+UxJ6vmF4r15itN3p+Mlk+M1OCdGSdfwO3V161Jt3p/OLSEcRlaT3G13O6Su7JFzbzRj9nXzN6GfhXev4/YhSWtPXaDSa4YSjF32NRqMZJmiXTY1Goxk+KMAZokbaVNCLvkaj0SSjlDbk7m/KRpbyzwdu4LDSDHLnfh0zECSYX8qII06ndEweh04p4thJRRw5ModxOX781WuIrX2elv8u5/TSTArH5lIwKZ+C6WPImzoe/7hpSPlE7LxRNNg+atotNjeEyfUbnQKw8jP9BIsyyCrJILM0k2BJPhnFeWSUF9LwwJJOAVhdDZFimIm2ui5MU9g13DZF3ACspvYYrWF3uzXsGXHDFlbMJquoqFMAlpEUlGX6xDXuehWwNq/Y1ikAK97s+L7dkR2zOCdtlwAsv+kabf1GvOpVx7Ydiybup7dqV37D6BSAlZxRs1N/D+/fHaZnsd2d4XZPDa09GW+14Xb4onRwlkaj0Qwj9KKv0Wg0wwkdkavRaDTDh30UkZtKfREROVFEFie1sIhc4B17REQ2Jh2bmcp1h8STfkmohrQbLuO1j3ZwzLd/3yn4qtwXxty+iujq16l/bjXrVm+ldnUddVWtVIYsrv37NxPBV1beSGraLWraLDY1tLN5Y0f1q4aGMLeNy0sEX2WUZJFRkk9GWQHB4gKM/BJ8hWUYecU4wVzC/3tnpzkma/iG3610Zfj8GL4A725p6BR8Fdfw2z0N34o5WFE7Ue0qtzAjEXwVT7IWD6pK8xkEAz533zR4v6U+EXwV1/CTq1y5zX1qKUj3dwq+MrtsG4Kr+ZsdwVlxutPgk/t8ZufgK0M69PvkoK2extodBp21912Cqvo0WtL7djPmLuf2cez+1vCT0Xr+wKLYZ3768foid4jILd7+9zrNRanXgZngfkkA64GXkk75rlLqqb5cdEgs+hqNRrPPUApn33jvnI+bqgbc+iJv0GXR78JFwItKqfa9uaiWdzQajSYJpdwn/VTaXpJyfRGPy4DHuvT9XESWisjdIpKWykX1k75Go9F0oQ9VsYpEZFHS/v1eLRAAROQVoKyb9/2g0/V6qS/ipaI/BJif1H0r7pdFALf2yPeAn/Q24SGx6Fdua+RP29YSNIVXT1dEVr1A/eNrqFu1jQ2r66jZ0cqOsE1t1KLVcgglfQMvP/SzbGwMsXlNOxU1a9hc20ZTY5i25jChlijhtvZE4rTZ3zyZYGkxZn4xZn5JQr93grk4adm0OkJbTNEeczB8gW71e8MfwBdwk6UZvgBmWpDXV+3sUb+3om7xk+QiKNNnjSDgM8gImAR8ZkK/j2v6yYVPom1NwK76fbKuD24BlPygv5N+7zeMRLGT7oqf9KbpJxMw4gVOOuv38Z+S3RVASRUz6U1d3743/vQ9vVdL5sMc1aen+Fql1Oyeh1Kn9HRMRPpSX+QS4BmlVCxp7PivhIiI/Bn4TioT1vKORqPRJOP56afS9pK+1Be5nC7SjvdFgbhPVBcAy1O56JB40tdoNJp9hWKfJVzrtr6IiMwGvqqU+pK3Pw4YDbzZ5f1/E5Fi3B+ni4GvpnJRvehrNBpNMkphRwd+0VdK1dFNfRGl1CLgS0n7m4CR3Zx30p5cVy/6Go1Gk4RS4CidhmG/UpyTxs1fPIaC6eO4a861NMRsWi2HqBcRZ4prSMzyGYxI91MQMChO85FREORL975He0uESFura7Bta8IKt+FYUaxIKFFdCiD7ijuw03NojTm0xRxClkMo5tBUb9EUaaE14lW8ilhkj5joGXADmIGgZ8BNS6pqZSaSo22paMD2DLVW1EYplah01bXKlXJsDh45vVN1q0RLSpLmNwxMASvcBnQ22EL3Va7yg/5uDbZdE6OlEkS1a8K1+BidDbY9VbrqC0L3Rtc9qZbVddxU0QnThhe2XvQ1Go1meKCAAzjfml70NRqNpiv6SV+j0WiGCY4iIR0fiAyJRd8ZM4H3r/wVG+vbCfAUEzP9FARMsgszyCgKklmaSWZJNhllhWSU5BMoLMAsLMfML2bNl/6Z0OyTSSRH8wKoTF+ApzZGaYrscLV7L0FaUyhGKGrRErYIJQVYlU2dkdDs4wVPDNPb9wqcxIOpFsxf2kmz7y5BWnIw1bTy7IRm7zPdV1fL79iOJ0uL2yW60l1fTpqvk2YfT5DWtcBJXL/uS2I0nyk9FjnZ24IkZpcB+rvAiTum1uw1HWh5R6PRaIYJCqXlHY1GoxkuaEOuRqPRDDP0or+fWbe5mi9/49c4sSitH94PmfluErT0HGK+IO0xh5ClaIw5VEZtmiIWTeEYrVGbYH7pLonQzDT31Rfwu3q851t/97MrvWRonROgObaDbVmuHu/51Z9+7qyE73zXJGgJH3vTwG8IL/xlY6dEaMlaeXd+9ZMLMnebCC25WIkdDaX0b6gcm6yAq7p3TYIG3fvV94VACrr7nvrVJxdk6U/6ouNrjX74oJT23tFoNJphg0J772g0Gs2wQWv6Go1GM8zQ8o5Go9EME1xNf3/PYuAYEou+GUinZMY8TJ/ByfMFK1qPFavxAqVsbEvhWE6iGpVyFLZl4VhRTvvs2Z5x1SToNztVn+qa0Oy2Hz6SFCTldFt9Ks7lR5yHIexiaO3O8Bpuqk28L5WApzG5bqnLVKpP9SWAKtPvjtSdTXJvA578ZucB+tPuaQ6QFVUbZzU9oZ/0NRqNZpiggH1SQmU/oRd9jUajSUKhtPeORqPRDBdc7x296O9XDh5bwDu/OweA3Llf79N7H7nv4pTP/XbN1pTPnTc6O+Vzu0v4tjtKMgfmf0uGf0/LmPSObyCyoHlo7V2zTznADbkDtwrsBhE5Q0TWiMh6Ebllf8xBo9FouiP+pJ9K2xtE5GIRWSEijlcMvafzul0vRWS8iHzg9f9DRAKpXHefL/oiYgL3AGcCM4DLRWTGvp6HRqPR9IStUmt7yXLgQmBBTyf0sl7eCdytlJoENADXpHLR/fGkPwdYr5SqUEpFgceB8/fDPDQajWYXHNw0DKm0vUEptUoptaaX07pdL8X13z4JeMo771HgglSuK2ofGyxE5CLgDKXUl7z9zwNHKaWu73LetcC13u7BuN+KBwpFQG2vZw0dDrT7gQPvnobT/YxVShXv6cAi8l9v/FRIB8JJ+/crpe7v4/XeAL6jlFrUzbFu10vgR8D73lM+IjIaeFEpdXBv1xu0hlzvH+5+ABFZpJTqUfMaauj7GfwcaPek7yd1lFJn9NdYIvIKUNbNoR8opf7dX9fpC/tj0a8ERiftj/L6NBqN5oBCKXXKXg7R03pZB+SJiE8pZdGHdXR/aPoLgcme5TkAXAY8ux/modFoNIOdbtdL5eryrwMXeeddBaT0y2GfL/ret9L1wHxgFfCEUmpFL2/rk0Y2BND3M/g50O5J388gQ0Q+LSLbgLnAf0Rkvtc/QkRegF7Xy+8B3xaR9UAh8FBK193XhlyNRqPR7D/2S3CWRqPRaPYPetHXaDSaYcSgXvSHaroGEXlYRHaKyPKkvgIReVlE1nmv+V6/iMjvvHtcKiKz9t/Mu0dERovI6yKy0gsbv8HrH5L3JCLpIvKhiCzx7ufHXn+3Ye0ikubtr/eOj9uvN9ADImKKyCci8ry3P9TvZ5OILBORxSKyyOsbkp+5wcSgXfSHeLqGR4Cuvr63AK8qpSYDr3r74N7fZK9dC/xxH82xL1jATUqpGcDRwHXe/4uhek8R4CSl1GHATOAMETmansParwEavP67vfMGIzfgGvviDPX7AThRKTUzySd/qH7mBg9KqUHZcC3a85P2bwVu3d/z6sP8xwHLk/bXAOXedjmwxtv+E3B5d+cN1obrGnbqgXBPQAbwMW6UYy3g8/oTnz9cz4m53rbPO0/299y73Mco3EXwJOB53OJlQ/Z+vLltAoq69A35z9z+boP2SR8YCSTnOt7m9Q1VSpVS273tHUCptz2k7tOTAg4HPmAI35MnhSwGdgIvAxuARuW6yEHnOSfuxzvehOsiN5j4DXAzHUWfChna9wNuwsuXROQjLy0LDOHP3GBh0KZhOJBRSikRGXK+siKSBTwN3KiUapakRPdD7Z6UUjYwU0TygGeAaft3RnuOiJwD7FRKfSQiJ+zn6fQnn1JKVYpICfCyiKxOPjjUPnODhcH8pH+gpWuoFpFyAO91p9c/JO5TRPy4C/7flFL/9LqH9D0BKKUacSMb5+KFtXuHkuecuB/veC5uGPxgYR5wnohsws3CeBLwW4bu/QCglKr0XnfifjHP4QD4zO1vBvOif6Cla3gWN1QaOodMPwtc6XkfHA00Jf18HRSI+0j/ELBKKXVX0qEheU8iUuw94SMiQVz7xCp6DmtPvs+LgNeUJxwPBpRStyqlRimlxuH+nbymlPocQ/R+AEQkU0Sy49vAabiZdofkZ25Qsb+NCrtrwFnAWly99Qf7ez59mPdjwHYghqstXoOrmb4KrANeAQq8cwXXS2kDsAyYvb/n3839fApXX10KLPbaWUP1noBDgU+8+1kO3O71TwA+BNYDTwJpXn+6t7/eOz5hf9/Dbu7tBOD5oX4/3tyXeG1F/O9/qH7mBlPTaRg0Go1mGDGY5R2NRqPR9DN60ddoNJphhF70NRqNZhihF32NRqMZRuhFX6PRaIYRetHX7HdExPYyKa7wMl/eJCJ7/NkUke8nbY+TpGynGs1wRy/6msFASLmZFA/CDZQ6E/jhXoz3/d5P0WiGJ3rR1wwqlBtyfy1wvRddaYrI/4jIQi9P+lcAROQEEVkgIv8Rt+bCfSJiiMgdQND75fA3b1hTRB7wfkm85EXhajTDEr3oawYdSqkKwARKcKOZm5RSRwJHAl8WkfHeqXOAb+DWW5gIXKiUuoWOXw6f886bDNzj/ZJoBD6zz25Goxlk6EVfM9g5DTenymLcdM6FuIs4wIdKqQrlZsx8DDddRHdsVEot9rY/wq11oNEMS3RqZc2gQ0QmADZuBkUBvqGUmt/lnBNw8wEl01NOkUjStg1oeUczbNFP+ppBhYgUA/cBf1BuYqj5wNe81M6IyBQv6yLAHC8LqwFcCrzt9cfi52s0ms7oJ33NYCDoyTd+3Hq8/wfEUzg/iCvHfOyleK4BLvCOLQT+AEzCTSP8jNd/P7BURD4GfjDw09dohg46y6ZmSOLJO99RSp2zn6ei0QwptLyj0Wg0wwj9pK/RaDTDCP2kr9FoNMMIvehrNBrNMEIv+hqNRjOM0Iu+RqPRDCP0oq/RaDTDiP8PXC8ozdJ2JrkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample_pos_encoding = PositionalEncoding(50, 512)\n",
    "\n",
    "plt.pcolormesh(sample_pos_encoding.pos_encoding.numpy()[0], cmap='RdBu')\n",
    "plt.xlabel('Depth')\n",
    "plt.xlim((0, 512))\n",
    "plt.ylabel('Position')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "id": "entire-laptop",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 스케일드 닷 프로덕트 어텐션 함수\n",
    "def scaled_dot_product_attention(query, key, value, mask):\n",
    "  # 어텐션 가중치는 Q와 K의 닷 프로덕트\n",
    "  matmul_qk = tf.matmul(query, key, transpose_b=True)\n",
    "\n",
    "  # 가중치를 정규화\n",
    "  depth = tf.cast(tf.shape(key)[-1], tf.float32)\n",
    "  logits = matmul_qk / tf.math.sqrt(depth)\n",
    "\n",
    "  # 패딩에 마스크 추가\n",
    "  if mask is not None:\n",
    "    logits += (mask * -1e9)\n",
    "\n",
    "  # softmax적용\n",
    "  attention_weights = tf.nn.softmax(logits, axis=-1)\n",
    "\n",
    "  # 최종 어텐션은 가중치와 V의 닷 프로덕트\n",
    "  output = tf.matmul(attention_weights, value)\n",
    "  return output\n",
    "\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "id": "grave-bridge",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "\n",
    "  def __init__(self, d_model, num_heads, name=\"multi_head_attention\"):\n",
    "    super(MultiHeadAttention, self).__init__(name=name)\n",
    "    self.num_heads = num_heads\n",
    "    self.d_model = d_model\n",
    "\n",
    "    assert d_model % self.num_heads == 0\n",
    "\n",
    "    self.depth = d_model // self.num_heads\n",
    "\n",
    "    self.query_dense = tf.keras.layers.Dense(units=d_model)\n",
    "    self.key_dense = tf.keras.layers.Dense(units=d_model)\n",
    "    self.value_dense = tf.keras.layers.Dense(units=d_model)\n",
    "\n",
    "    self.dense = tf.keras.layers.Dense(units=d_model)\n",
    "\n",
    "  def split_heads(self, inputs, batch_size):\n",
    "    inputs = tf.reshape(\n",
    "        inputs, shape=(batch_size, -1, self.num_heads, self.depth))\n",
    "    return tf.transpose(inputs, perm=[0, 2, 1, 3])\n",
    "\n",
    "  def call(self, inputs):\n",
    "    query, key, value, mask = inputs['query'], inputs['key'], inputs[\n",
    "        'value'], inputs['mask']\n",
    "    batch_size = tf.shape(query)[0]\n",
    "\n",
    "    # Q, K, V에 각각 Dense를 적용합니다\n",
    "    query = self.query_dense(query)\n",
    "    key = self.key_dense(key)\n",
    "    value = self.value_dense(value)\n",
    "\n",
    "    # 병렬 연산을 위한 머리를 여러 개 만듭니다\n",
    "    query = self.split_heads(query, batch_size)\n",
    "    key = self.split_heads(key, batch_size)\n",
    "    value = self.split_heads(value, batch_size)\n",
    "\n",
    "    # 스케일드 닷 프로덕트 어텐션 함수\n",
    "    scaled_attention = scaled_dot_product_attention(query, key, value, mask)\n",
    "\n",
    "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
    "\n",
    "    # 어텐션 연산 후에 각 결과를 다시 연결(concatenate)합니다\n",
    "    concat_attention = tf.reshape(scaled_attention,\n",
    "                                  (batch_size, -1, self.d_model))\n",
    "\n",
    "    # 최종 결과에도 Dense를 한 번 더 적용합니다\n",
    "    outputs = self.dense(concat_attention)\n",
    "\n",
    "    return outputs\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "id": "great-thesaurus",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def create_padding_mask(x):\n",
    "  mask = tf.cast(tf.math.equal(x, 0), tf.float32)\n",
    "  # (batch_size, 1, 1, sequence length)\n",
    "  return mask[:, tf.newaxis, tf.newaxis, :]\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "id": "miniature-stake",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[[[0. 0. 1. 0. 1.]]]\n",
      "\n",
      "\n",
      " [[[1. 1. 1. 0. 0.]]]], shape=(2, 1, 1, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(create_padding_mask(tf.constant([[1, 2, 0, 3, 0], [0, 0, 0, 4, 5]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "id": "natural-empty",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def create_look_ahead_mask(x):\n",
    "  seq_len = tf.shape(x)[1]\n",
    "  look_ahead_mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)\n",
    "  padding_mask = create_padding_mask(x)\n",
    "  return tf.maximum(look_ahead_mask, padding_mask)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "id": "direct-bandwidth",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[[[0. 1. 1. 1. 1.]\n",
      "   [0. 0. 1. 1. 1.]\n",
      "   [0. 0. 0. 1. 1.]\n",
      "   [0. 0. 0. 0. 1.]\n",
      "   [0. 0. 0. 0. 0.]]]], shape=(1, 1, 5, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(create_look_ahead_mask(tf.constant([[1, 2, 3, 4, 5]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "id": "earned-working",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[[[1. 1. 1. 1. 1.]\n",
      "   [1. 0. 1. 1. 1.]\n",
      "   [1. 0. 0. 1. 1.]\n",
      "   [1. 0. 0. 0. 1.]\n",
      "   [1. 0. 0. 0. 0.]]]], shape=(1, 1, 5, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(create_look_ahead_mask(tf.constant([[0, 5, 1, 5, 5]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "id": "demonstrated-denmark",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 인코더 하나의 레이어를 함수로 구현.\n",
    "# 이 하나의 레이어 안에는 두 개의 서브 레이어가 존재합니다.\n",
    "def encoder_layer(units, d_model, num_heads, dropout, name=\"encoder_layer\"):\n",
    "  inputs = tf.keras.Input(shape=(None, d_model), name=\"inputs\")\n",
    "\n",
    "  # 패딩 마스크 사용\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
    "\n",
    "  # 첫 번째 서브 레이어 : 멀티 헤드 어텐션 수행 (셀프 어텐션)\n",
    "  attention = MultiHeadAttention(\n",
    "      d_model, num_heads, name=\"attention\")({\n",
    "          'query': inputs,\n",
    "          'key': inputs,\n",
    "          'value': inputs,\n",
    "          'mask': padding_mask\n",
    "      })\n",
    "\n",
    "  # 어텐션의 결과는 Dropout과 Layer Normalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  attention = tf.keras.layers.Dropout(rate=dropout)(attention)\n",
    "  attention = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(inputs + attention)\n",
    "\n",
    "  # 두 번째 서브 레이어 : 2개의 완전연결층\n",
    "  outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention)\n",
    "  outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
    "\n",
    "  # 완전연결층의 결과는 Dropout과 LayerNormalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
    "  outputs = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(attention + outputs)\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, padding_mask], outputs=outputs, name=name)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "id": "electrical-dealing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def encoder(vocab_size,\n",
    "            num_layers,\n",
    "            units,\n",
    "            d_model,\n",
    "            num_heads,\n",
    "            dropout,\n",
    "            name=\"encoder\"):\n",
    "  inputs = tf.keras.Input(shape=(None,), name=\"inputs\")\n",
    "\n",
    "  # 패딩 마스크 사용\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
    "\n",
    "  # 임베딩 레이어\n",
    "  embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)\n",
    "  embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
    "\n",
    "  # 포지셔널 인코딩\n",
    "  embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)\n",
    "\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)\n",
    "\n",
    "  # num_layers만큼 쌓아올린 인코더의 층.\n",
    "  for i in range(num_layers):\n",
    "    outputs = encoder_layer(\n",
    "        units=units,\n",
    "        d_model=d_model,\n",
    "        num_heads=num_heads,\n",
    "        dropout=dropout,\n",
    "        name=\"encoder_layer_{}\".format(i),\n",
    "    )([outputs, padding_mask])\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, padding_mask], outputs=outputs, name=name)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "id": "united-helmet",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 디코더 하나의 레이어를 함수로 구현.\n",
    "# 이 하나의 레이어 안에는 세 개의 서브 레이어가 존재합니다.\n",
    "def decoder_layer(units, d_model, num_heads, dropout, name=\"decoder_layer\"):\n",
    "  inputs = tf.keras.Input(shape=(None, d_model), name=\"inputs\")\n",
    "  enc_outputs = tf.keras.Input(shape=(None, d_model), name=\"encoder_outputs\")\n",
    "  look_ahead_mask = tf.keras.Input(\n",
    "      shape=(1, None, None), name=\"look_ahead_mask\")\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')\n",
    "\n",
    "  # 첫 번째 서브 레이어 : 멀티 헤드 어텐션 수행 (셀프 어텐션)\n",
    "  attention1 = MultiHeadAttention(\n",
    "      d_model, num_heads, name=\"attention_1\")(inputs={\n",
    "          'query': inputs,\n",
    "          'key': inputs,\n",
    "          'value': inputs,\n",
    "          'mask': look_ahead_mask\n",
    "      })\n",
    "\n",
    "  # 멀티 헤드 어텐션의 결과는 LayerNormalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  attention1 = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(attention1 + inputs)\n",
    "\n",
    "  # 두 번째 서브 레이어 : 마스크드 멀티 헤드 어텐션 수행 (인코더-디코더 어텐션)\n",
    "  attention2 = MultiHeadAttention(\n",
    "      d_model, num_heads, name=\"attention_2\")(inputs={\n",
    "          'query': attention1,\n",
    "          'key': enc_outputs,\n",
    "          'value': enc_outputs,\n",
    "          'mask': padding_mask\n",
    "      })\n",
    "\n",
    "  # 마스크드 멀티 헤드 어텐션의 결과는\n",
    "  # Dropout과 LayerNormalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  attention2 = tf.keras.layers.Dropout(rate=dropout)(attention2)\n",
    "  attention2 = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(attention2 + attention1)\n",
    "\n",
    "  # 세 번째 서브 레이어 : 2개의 완전연결층\n",
    "  outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention2)\n",
    "  outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
    "\n",
    "  # 완전연결층의 결과는 Dropout과 LayerNormalization 수행\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
    "  outputs = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(outputs + attention2)\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],\n",
    "      outputs=outputs,\n",
    "      name=name)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "id": "going-latitude",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def decoder(vocab_size,\n",
    "            num_layers,\n",
    "            units,\n",
    "            d_model,\n",
    "            num_heads,\n",
    "            dropout,\n",
    "            name='decoder'):\n",
    "  inputs = tf.keras.Input(shape=(None,), name='inputs')\n",
    "  enc_outputs = tf.keras.Input(shape=(None, d_model), name='encoder_outputs')\n",
    "  look_ahead_mask = tf.keras.Input(\n",
    "      shape=(1, None, None), name='look_ahead_mask')\n",
    "\n",
    "  # 패딩 마스크\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')\n",
    "  \n",
    "  # 임베딩 레이어\n",
    "  embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)\n",
    "  embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
    "\n",
    "  # 포지셔널 인코딩\n",
    "  embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)\n",
    "\n",
    "  # Dropout이라는 훈련을 돕는 테크닉을 수행\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)\n",
    "\n",
    "  for i in range(num_layers):\n",
    "    outputs = decoder_layer(\n",
    "        units=units,\n",
    "        d_model=d_model,\n",
    "        num_heads=num_heads,\n",
    "        dropout=dropout,\n",
    "        name='decoder_layer_{}'.format(i),\n",
    "    )(inputs=[outputs, enc_outputs, look_ahead_mask, padding_mask])\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],\n",
    "      outputs=outputs,\n",
    "      name=name)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "prepared-degree",
   "metadata": {},
   "source": [
    "**Step 3. SubwordTextEncoder 사용하기**<br>\n",
    "한국어 데이터는 형태소 분석기를 사용하여 토크나이징을 해야 한다고 많은 분이 알고 있습니다. 하지만 여기서는 형태소 분석기가 아닌 위 실습에서 사용했던 내부 단어 토크나이저인 SubwordTextEncoder를 그대로 사용해보세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "id": "registered-quality",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q</th>\n",
       "      <th>A</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12시 땡!</td>\n",
       "      <td>하루가 또 가네요.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1지망 학교 떨어졌어</td>\n",
       "      <td>위로해 드립니다.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3박4일 놀러가고 싶다</td>\n",
       "      <td>여행은 언제나 좋죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3박4일 정도 놀러가고 싶다</td>\n",
       "      <td>여행은 언제나 좋죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PPL 심하네</td>\n",
       "      <td>눈살이 찌푸려지죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11818</th>\n",
       "      <td>훔쳐보는 것도 눈치 보임.</td>\n",
       "      <td>티가 나니까 눈치가 보이는 거죠!</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11819</th>\n",
       "      <td>훔쳐보는 것도 눈치 보임.</td>\n",
       "      <td>훔쳐보는 거 티나나봐요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11820</th>\n",
       "      <td>흑기사 해주는 짝남.</td>\n",
       "      <td>설렜겠어요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11821</th>\n",
       "      <td>힘든 연애 좋은 연애라는게 무슨 차이일까?</td>\n",
       "      <td>잘 헤어질 수 있는 사이 여부인 거 같아요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11822</th>\n",
       "      <td>힘들어서 결혼할까봐</td>\n",
       "      <td>도피성 결혼은 하지 않길 바라요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11823 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Q                         A  label\n",
       "0                       12시 땡!                하루가 또 가네요.      0\n",
       "1                  1지망 학교 떨어졌어                 위로해 드립니다.      0\n",
       "2                 3박4일 놀러가고 싶다               여행은 언제나 좋죠.      0\n",
       "3              3박4일 정도 놀러가고 싶다               여행은 언제나 좋죠.      0\n",
       "4                      PPL 심하네                눈살이 찌푸려지죠.      0\n",
       "...                        ...                       ...    ...\n",
       "11818           훔쳐보는 것도 눈치 보임.        티가 나니까 눈치가 보이는 거죠!      2\n",
       "11819           훔쳐보는 것도 눈치 보임.             훔쳐보는 거 티나나봐요.      2\n",
       "11820              흑기사 해주는 짝남.                    설렜겠어요.      2\n",
       "11821  힘든 연애 좋은 연애라는게 무슨 차이일까?  잘 헤어질 수 있는 사이 여부인 거 같아요.      2\n",
       "11822               힘들어서 결혼할까봐        도피성 결혼은 하지 않길 바라요.      2\n",
       "\n",
       "[11823 rows x 3 columns]"
      ]
     },
     "execution_count": 424,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.getenv('HOME') + '/aiffel/ChatbotData .csv' \n",
    "data = pd.read_csv(path) \n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "id": "ruled-elder",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11823\n"
     ]
    }
   ],
   "source": [
    "# 사용할 샘플의 최대 개수\n",
    "MAX_SAMPLES = 11823\n",
    "print(MAX_SAMPLES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "id": "biological-therapist",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 전처리 함수\n",
    "def preprocess_sentence(sentence):\n",
    "  sentence = sentence.lower().strip()\n",
    "\n",
    "  # 단어와 구두점(punctuation) 사이의 거리를 만듭니다.\n",
    "  # 예를 들어서 \"I am a student.\" => \"I am a student .\"와 같이\n",
    "  # student와 온점 사이에 거리를 만듭니다.\n",
    "  sentence = re.sub(r\"([?.!,])\", r\" \\1 \", sentence)\n",
    "  sentence = re.sub(r'[\" \"]+', \" \", sentence)\n",
    "\n",
    "  # (a-z, A-Z, \".\", \"?\", \"!\", \",\")를 제외한 모든 문자를 공백인 ' '로 대체합니다.\n",
    "  sentence = re.sub(r\"[^a-zA-Z?.!,]+\", \" \", sentence)\n",
    "  sentence = sentence.strip()\n",
    "  return sentence\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 427,
   "id": "iraqi-nutrition",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 질문과 답변의 쌍인 데이터셋을 구성하기 위한 데이터 로드 함수\n",
    "def load_conversations():\n",
    "  id2line = {}\n",
    "  with open(path_to_dataset, errors='ignore') as file:\n",
    "    lines = file.readlines()\n",
    "  for line in lines:\n",
    "    parts = line.replace('\\n', '').split(' +++$+++ ')\n",
    "    id2line[parts[0]] = parts[4]\n",
    "\n",
    "  inputs, outputs = [], []\n",
    "  with open(path_to_dataset, 'r') as file:\n",
    "    lines = file.readlines()\n",
    "\n",
    "  for line in lines:\n",
    "    parts = line.replace('\\n', '').split(' +++$+++ ')\n",
    "    conversation = [line[1:-1] for line in parts[3][1:-1].split(', ')]\n",
    "\n",
    "    for i in range(len(conversation) - 1):\n",
    "      # 전처리 함수를 질문에 해당되는 inputs와 답변에 해당되는 outputs에 적용.\n",
    "      inputs.append(preprocess_sentence(id2line[conversation[i]]))\n",
    "      outputs.append(preprocess_sentence(id2line[conversation[i + 1]]))\n",
    "\n",
    "      if len(inputs) >= MAX_SAMPLES:\n",
    "        return inputs, outputs\n",
    "  return inputs, outputs\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "id": "abroad-perry",
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = data['Q'].map(preprocess_sentence) \n",
    "answers = data['A'].map(preprocess_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "atlantic-first",
   "metadata": {},
   "source": [
    "**Step 4. 모델 구성하기**\n",
    "위 실습 내용을 참고하여 트랜스포머 모델을 구현합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "id": "intensive-worst",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "살짝 오래 걸릴 수 있어요. 스트레칭 한 번 해볼까요? 👐\n",
      "슝=3 \n"
     ]
    }
   ],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "print(\"살짝 오래 걸릴 수 있어요. 스트레칭 한 번 해볼까요? 👐\")\n",
    "\n",
    "# 질문과 답변 데이터셋에 대해서 Vocabulary 생성. (Tensorflow 2.3.0 이상) (클라우드는 2.4 입니다)\n",
    "tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(questions + answers, target_vocab_size=2**13)\n",
    "print(\"슝=3 \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "id": "certain-tolerance",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 시작 토큰과 종료 토큰에 고유한 정수를 부여합니다.\n",
    "START_TOKEN, END_TOKEN = [tokenizer.vocab_size], [tokenizer.vocab_size + 1]\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "id": "relevant-earthquake",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START_TOKEN의 번호 : [346]\n",
      "END_TOKEN의 번호 : [347]\n"
     ]
    }
   ],
   "source": [
    "print('START_TOKEN의 번호 :' ,[tokenizer.vocab_size])\n",
    "print('END_TOKEN의 번호 :' ,[tokenizer.vocab_size + 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "id": "emotional-victory",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348\n"
     ]
    }
   ],
   "source": [
    "# 시작 토큰과 종료 토큰을 고려하여 +2를 하여 단어장의 크기를 산정합니다.\n",
    "VOCAB_SIZE = tokenizer.vocab_size + 2\n",
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "id": "incoming-living",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정수 인코딩 후의 21번째 질문 샘플: []\n",
      "정수 인코딩 후의 21번째 답변 샘플: [136]\n"
     ]
    }
   ],
   "source": [
    "# 임의의 22번째 샘플에 대해서 정수 인코딩 작업을 수행.\n",
    "# 각 토큰을 고유한 정수로 변환\n",
    "print('정수 인코딩 후의 21번째 질문 샘플: {}'.format(tokenizer.encode(questions[21])))\n",
    "print('정수 인코딩 후의 21번째 답변 샘플: {}'.format(tokenizer.encode(answers[21])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "id": "written-trial",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "# 샘플의 최대 허용 길이 또는 패딩 후의 최종 길이\n",
    "MAX_LENGTH = 40\n",
    "print(MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "id": "prescribed-lodging",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "# 정수 인코딩, 최대 길이를 초과하는 샘플 제거, 패딩\n",
    "def tokenize_and_filter(inputs, outputs):\n",
    "  tokenized_inputs, tokenized_outputs = [], []\n",
    "  \n",
    "  for (sentence1, sentence2) in zip(inputs, outputs):\n",
    "    # 정수 인코딩 과정에서 시작 토큰과 종료 토큰을 추가\n",
    "    sentence1 = START_TOKEN + tokenizer.encode(sentence1) + END_TOKEN\n",
    "    sentence2 = START_TOKEN + tokenizer.encode(sentence2) + END_TOKEN\n",
    "\n",
    "    # 최대 길이 40 이하인 경우에만 데이터셋으로 허용\n",
    "    if len(sentence1) <= MAX_LENGTH and len(sentence2) <= MAX_LENGTH:\n",
    "      tokenized_inputs.append(sentence1)\n",
    "      tokenized_outputs.append(sentence2)\n",
    "  \n",
    "  # 최대 길이 40으로 모든 데이터셋을 패딩\n",
    "  tokenized_inputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "      tokenized_inputs, maxlen=MAX_LENGTH, padding='post')\n",
    "  tokenized_outputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "      tokenized_outputs, maxlen=MAX_LENGTH, padding='post')\n",
    "  \n",
    "  return tokenized_inputs, tokenized_outputs\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "id": "solar-request",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "단어장의 크기 : 348\n",
      "필터링 후의 질문 샘플 개수: 11823\n",
      "필터링 후의 답변 샘플 개수: 11823\n"
     ]
    }
   ],
   "source": [
    "questions, answers = tokenize_and_filter(questions, answers)\n",
    "print('단어장의 크기 :',(VOCAB_SIZE))\n",
    "print('필터링 후의 질문 샘플 개수: {}'.format(len(questions)))\n",
    "print('필터링 후의 답변 샘플 개수: {}'.format(len(answers)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "id": "cooperative-communication",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 64\n",
    "BUFFER_SIZE = 20000\n",
    "\n",
    "# 디코더는 이전의 target을 다음의 input으로 사용합니다.\n",
    "# 이에 따라 outputs에서는 START_TOKEN을 제거하겠습니다.\n",
    "dataset = tf.data.Dataset.from_tensor_slices((\n",
    "    {\n",
    "        'inputs': questions,\n",
    "        'dec_inputs': answers[:, :-1]\n",
    "    },\n",
    "    {\n",
    "        'outputs': answers[:, 1:]\n",
    "    },\n",
    "))\n",
    "\n",
    "dataset = dataset.cache()\n",
    "dataset = dataset.shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE)\n",
    "dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "id": "unavailable-craft",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def transformer(vocab_size,\n",
    "                num_layers,\n",
    "                units,\n",
    "                d_model,\n",
    "                num_heads,\n",
    "                dropout,\n",
    "                name=\"transformer\"):\n",
    "  inputs = tf.keras.Input(shape=(None,), name=\"inputs\")\n",
    "  dec_inputs = tf.keras.Input(shape=(None,), name=\"dec_inputs\")\n",
    "\n",
    "  # 인코더에서 패딩을 위한 마스크\n",
    "  enc_padding_mask = tf.keras.layers.Lambda(\n",
    "      create_padding_mask, output_shape=(1, 1, None),\n",
    "      name='enc_padding_mask')(inputs)\n",
    "\n",
    "  # 디코더에서 미래의 토큰을 마스크 하기 위해서 사용합니다.\n",
    "  # 내부적으로 패딩 마스크도 포함되어져 있습니다.\n",
    "  look_ahead_mask = tf.keras.layers.Lambda(\n",
    "      create_look_ahead_mask,\n",
    "      output_shape=(1, None, None),\n",
    "      name='look_ahead_mask')(dec_inputs)\n",
    "\n",
    "  # 두 번째 어텐션 블록에서 인코더의 벡터들을 마스킹\n",
    "  # 디코더에서 패딩을 위한 마스크\n",
    "  dec_padding_mask = tf.keras.layers.Lambda(\n",
    "      create_padding_mask, output_shape=(1, 1, None),\n",
    "      name='dec_padding_mask')(inputs)\n",
    "\n",
    "  # 인코더\n",
    "  enc_outputs = encoder(\n",
    "      vocab_size=vocab_size,\n",
    "      num_layers=num_layers,\n",
    "      units=units,\n",
    "      d_model=d_model,\n",
    "      num_heads=num_heads,\n",
    "      dropout=dropout,\n",
    "  )(inputs=[inputs, enc_padding_mask])\n",
    "\n",
    "  # 디코더\n",
    "  dec_outputs = decoder(\n",
    "      vocab_size=vocab_size,\n",
    "      num_layers=num_layers,\n",
    "      units=units,\n",
    "      d_model=d_model,\n",
    "      num_heads=num_heads,\n",
    "      dropout=dropout,\n",
    "  )(inputs=[dec_inputs, enc_outputs, look_ahead_mask, dec_padding_mask])\n",
    "\n",
    "  # 완전연결층\n",
    "  outputs = tf.keras.layers.Dense(units=vocab_size, name=\"outputs\")(dec_outputs)\n",
    "\n",
    "  return tf.keras.Model(inputs=[inputs, dec_inputs], outputs=outputs, name=name)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "id": "ultimate-fourth",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"transformer\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "inputs (InputLayer)             [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dec_inputs (InputLayer)         [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "enc_padding_mask (Lambda)       (None, 1, 1, None)   0           inputs[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "encoder (Functional)            (None, None, 256)    1143296     inputs[0][0]                     \n",
      "                                                                 enc_padding_mask[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "look_ahead_mask (Lambda)        (None, 1, None, None 0           dec_inputs[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dec_padding_mask (Lambda)       (None, 1, 1, None)   0           inputs[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "decoder (Functional)            (None, None, 256)    1670656     dec_inputs[0][0]                 \n",
      "                                                                 encoder[0][0]                    \n",
      "                                                                 look_ahead_mask[0][0]            \n",
      "                                                                 dec_padding_mask[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "outputs (Dense)                 (None, None, 348)    89436       decoder[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,903,388\n",
      "Trainable params: 2,903,388\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "\n",
    "# 하이퍼파라미터\n",
    "NUM_LAYERS = 2 # 인코더와 디코더의 층의 개수\n",
    "D_MODEL = 256 # 인코더와 디코더 내부의 입, 출력의 고정 차원\n",
    "NUM_HEADS = 8 # 멀티 헤드 어텐션에서의 헤드 수 \n",
    "UNITS = 512 # 피드 포워드 신경망의 은닉층의 크기\n",
    "DROPOUT = 0.1 # 드롭아웃의 비율\n",
    "\n",
    "model = transformer(\n",
    "    vocab_size=VOCAB_SIZE,\n",
    "    num_layers=NUM_LAYERS,\n",
    "    units=UNITS,\n",
    "    d_model=D_MODEL,\n",
    "    num_heads=NUM_HEADS,\n",
    "    dropout=DROPOUT)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "id": "combined-edwards",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def loss_function(y_true, y_pred):\n",
    "  y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))\n",
    "  \n",
    "  loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "      from_logits=True, reduction='none')(y_true, y_pred)\n",
    "\n",
    "  mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)\n",
    "  loss = tf.multiply(loss, mask)\n",
    "\n",
    "  return tf.reduce_mean(loss)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "id": "color-australia",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "\n",
    "  def __init__(self, d_model, warmup_steps=4000):\n",
    "    super(CustomSchedule, self).__init__()\n",
    "\n",
    "    self.d_model = d_model\n",
    "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "\n",
    "    self.warmup_steps = warmup_steps\n",
    "\n",
    "  def __call__(self, step):\n",
    "    arg1 = tf.math.rsqrt(step)\n",
    "    arg2 = step * (self.warmup_steps**-1.5)\n",
    "\n",
    "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "id": "excellent-steal",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "learning_rate = CustomSchedule(D_MODEL)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(\n",
    "    learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)\n",
    "\n",
    "def accuracy(y_true, y_pred):\n",
    "  y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))\n",
    "  return tf.keras.metrics.sparse_categorical_accuracy(y_true, y_pred)\n",
    "\n",
    "model.compile(optimizer=optimizer, loss=loss_function, metrics=[accuracy])\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "id": "fewer-drunk",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "185/185 [==============================] - 21s 76ms/step - loss: 0.1726 - accuracy: 0.0220\n",
      "Epoch 2/20\n",
      "185/185 [==============================] - 14s 75ms/step - loss: 0.0120 - accuracy: 0.0492\n",
      "Epoch 3/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0104 - accuracy: 0.0493\n",
      "Epoch 4/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0100 - accuracy: 0.0493\n",
      "Epoch 5/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0094 - accuracy: 0.0493\n",
      "Epoch 6/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0092 - accuracy: 0.0494\n",
      "Epoch 7/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0094 - accuracy: 0.0494\n",
      "Epoch 8/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0095 - accuracy: 0.0494\n",
      "Epoch 9/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0096 - accuracy: 0.0493\n",
      "Epoch 10/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0090 - accuracy: 0.0494\n",
      "Epoch 11/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0100 - accuracy: 0.0492\n",
      "Epoch 12/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0097 - accuracy: 0.0492\n",
      "Epoch 13/20\n",
      "185/185 [==============================] - 14s 76ms/step - loss: 0.0100 - accuracy: 0.0493\n",
      "Epoch 14/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0098 - accuracy: 0.0492\n",
      "Epoch 15/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0099 - accuracy: 0.0492\n",
      "Epoch 16/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0099 - accuracy: 0.0493\n",
      "Epoch 17/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0092 - accuracy: 0.0493\n",
      "Epoch 18/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0094 - accuracy: 0.0493\n",
      "Epoch 19/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0093 - accuracy: 0.0493\n",
      "Epoch 20/20\n",
      "185/185 [==============================] - 14s 77ms/step - loss: 0.0094 - accuracy: 0.0494\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f5c103c0050>"
      ]
     },
     "execution_count": 443,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EPOCHS = 20\n",
    "model.fit(dataset, epochs=EPOCHS, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sorted-direction",
   "metadata": {},
   "source": [
    "**Step 5. 모델 평가하기**<br>\n",
    "Step 1에서 선택한 전처리 방법을 고려하여 입력된 문장에 대해서 대답을 얻는 예측 함수를 만듭니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "id": "outer-falls",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def decoder_inference(sentence):\n",
    "  sentence = preprocess_sentence(sentence)\n",
    "\n",
    "  # 입력된 문장을 정수 인코딩 후, 시작 토큰과 종료 토큰을 앞뒤로 추가.\n",
    "  # ex) Where have you been? → [[8331   86   30    5 1059    7 8332]]\n",
    "  sentence = tf.expand_dims(\n",
    "      START_TOKEN + tokenizer.encode(sentence) + END_TOKEN, axis=0)\n",
    "\n",
    "  # 디코더의 현재까지의 예측한 출력 시퀀스가 지속적으로 저장되는 변수.\n",
    "  # 처음에는 예측한 내용이 없음으로 시작 토큰만 별도 저장. ex) 8331\n",
    "  output_sequence = tf.expand_dims(START_TOKEN, 0)\n",
    "\n",
    "  # 디코더의 인퍼런스 단계\n",
    "  for i in range(MAX_LENGTH):\n",
    "    # 디코더는 최대 MAX_LENGTH의 길이만큼 다음 단어 예측을 반복합니다.\n",
    "    predictions = model(inputs=[sentence, output_sequence], training=False)\n",
    "    predictions = predictions[:, -1:, :]\n",
    "\n",
    "    # 현재 예측한 단어의 정수\n",
    "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "    # 만약 현재 예측한 단어가 종료 토큰이라면 for문을 종료\n",
    "    if tf.equal(predicted_id, END_TOKEN[0]):\n",
    "      break\n",
    "\n",
    "    # 예측한 단어들은 지속적으로 output_sequence에 추가됩니다.\n",
    "    # 이 output_sequence는 다시 디코더의 입력이 됩니다.\n",
    "    output_sequence = tf.concat([output_sequence, predicted_id], axis=-1)\n",
    "\n",
    "  return tf.squeeze(output_sequence, axis=0)\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "id": "incorrect-modern",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "슝=3\n"
     ]
    }
   ],
   "source": [
    "def sentence_generation(sentence):\n",
    "  # 입력 문장에 대해서 디코더를 동작 시켜 예측된 정수 시퀀스를 리턴받습니다.\n",
    "  prediction = decoder_inference(sentence)\n",
    "\n",
    "  # 정수 시퀀스를 다시 텍스트 시퀀스로 변환합니다.\n",
    "  predicted_sentence = tokenizer.decode(\n",
    "      [i for i in prediction if i < tokenizer.vocab_size])\n",
    "\n",
    "  print('입력 : {}'.format(sentence))\n",
    "  print('출력 : {}'.format(predicted_sentence))\n",
    "\n",
    "  return predicted_sentence\n",
    "print(\"슝=3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "id": "collectible-olive",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 힘들어서 결혼할까봐\n",
      "출력 : .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'.'"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation('힘들어서 결혼할까봐')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "id": "funky-victoria",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 흑기사 해주는 짝남.\n",
      "출력 : .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'.'"
      ]
     },
     "execution_count": 452,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation('흑기사 해주는 짝남.')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "refined-ordinance",
   "metadata": {},
   "source": [
    "**회고**<br>\n",
    "제가 스폐셜리스트가 되고 싶은 분야가 자연어처리 쪽이라서 오늘 노드를 진행하면서 아주 흥미로웠습니다.<br> \n",
    "챗봇도 5가지나 되는 유형이 있음을 알았고 딥러닝을 통한 자연어처리 기술의 발전을 보면서 유익한 시간이라고 느꼈습니다.<br>\n",
    "오늘 프로젝트는 트랜스포머 모델을 기반으로 한 인코더-디코더 구조를 바탕으로 챗봇을 제작하는 것인데, 엄청 많은 코퍼스를<br>\n",
    "학습시킨다면 더욱 다채로운 챗봇이 되지 않을까 라는 생각이 들었습니다. 아쉬운 점은 문장을 넣었을 때 출력으로 .만 출력이 되서<br>\n",
    "완벽하지 못한 모델이 나온 것 같아서 아쉽습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "premier-closing",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}