Transformer_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-28T03:23:05.218878Z",
     "start_time": "2021-09-28T03:22:25.914751Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
      "2.1.0\n",
      "sys.version_info(major=3, minor=7, micro=9, releaselevel='final', serial=0)\n",
      "matplotlib 3.3.2\n",
      "numpy 1.17.0\n",
      "pandas 1.1.3\n",
      "sklearn 0.23.2\n",
      "tensorflow 2.1.0\n",
      "tensorflow_core.python.keras.api._v2.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "# 设置gpu内存自增长\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "print(gpus)\n",
    "for gpu in gpus:\n",
    "    tf.config.experimental.set_memory_growth(gpu, True)\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.载入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:17:32.566273Z",
     "start_time": "2021-09-16T18:17:30.535095Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "example, info = tfds.load('ted_hrlr_translate/pt_to_en',\n",
    "                     with_info=True,\n",
    "                     as_supervised=True)\n",
    "# for data in datasets.take(5):\n",
    "#     print(data)\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:17:33.804388Z",
     "start_time": "2021-09-16T18:17:33.737334Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'e quando melhoramos a procura , tiramos a \\xc3\\xbanica vantagem da impress\\xc3\\xa3o , que \\xc3\\xa9 a serendipidade .'>, <tf.Tensor: shape=(), dtype=string, numpy=b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .'>)\n",
      "\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'mas e se estes fatores fossem ativos ?'>, <tf.Tensor: shape=(), dtype=string, numpy=b'but what if it were active ?'>)\n",
      "\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'mas eles n\\xc3\\xa3o tinham a curiosidade de me testar .'>, <tf.Tensor: shape=(), dtype=string, numpy=b\"but they did n't test for curiosity .\">)\n",
      "\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'e esta rebeldia consciente \\xc3\\xa9 a raz\\xc3\\xa3o pela qual eu , como agn\\xc3\\xb3stica , posso ainda ter f\\xc3\\xa9 .'>, <tf.Tensor: shape=(), dtype=string, numpy=b'and this conscious defiance is why i , as an agnostic , can still have faith .'>)\n",
      "\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b\"`` `` '' podem usar tudo sobre a mesa no meu corpo . ''\">, <tf.Tensor: shape=(), dtype=string, numpy=b'you can use everything on the table on me .'>)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train = example['train']\n",
    "for data in train.take(5):\n",
    "    print(data)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T01:24:20.604460Z",
     "start_time": "2021-09-16T01:22:02.423365Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "# 1、生成tokenizer，\n",
    "pt_tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
    "    (pt.numpy() for pt,en in train),\n",
    "    target_vocab_size =2**13\n",
    ")\n",
    "\n",
    "en_tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
    "    (en.numpy() for pt,en in train),\n",
    "    target_vocab_size =2**13\n",
    ")\n",
    "\n",
    "# 1.1 保存生成好的tokenizer\n",
    "en_tokenizer.save_to_file('./tokenizer/en_tokenizer_new')\n",
    "pt_tokenizer.save_to_file('./tokenizer/pt_tokenizer_new')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:17:37.500750Z",
     "start_time": "2021-09-16T18:17:37.410667Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "\n",
    "en_tokenizer = tfds.deprecated.text.SubwordTextEncoder.load_from_file('./tokenizer/en_tokenizer_new')\n",
    "pt_tokenizer = tfds.deprecated.text.SubwordTextEncoder.load_from_file('./tokenizer/en_tokenizer_new')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:17:47.635984Z",
     "start_time": "2021-09-16T18:17:47.473836Z"
    }
   },
   "outputs": [],
   "source": [
    "# 2、用tokenizer将字符串编码成数字\n",
    "def tokenizer_encoder(pt_sentence, en_sentence):\n",
    "    pt_sentence = [pt_tokenizer.vocab_size] + pt_tokenizer.encode(pt_sentence.numpy()) + [pt_tokenizer.vocab_size+1]\n",
    "    en_sentence = [en_tokenizer.vocab_size] + en_tokenizer.encode(en_sentence.numpy()) + [en_tokenizer.vocab_size+1]\n",
    "    return pt_sentence, en_sentence\n",
    "\n",
    "def tf_tokenizer_encoder(pt_sentence, en_sentence):\n",
    "    return tf.py_function(tokenizer_encoder,\n",
    "                         [pt_sentence,en_sentence],\n",
    "                         [tf.int64, tf.int64])\n",
    "\n",
    "# 3、datasets日常操作：filter、shuffle、batch、padding\n",
    "max_len = 40\n",
    "batch_size=64\n",
    "def max_len_filter(pt_sentence, en_sentence):\n",
    "    return tf.logical_and(tf.size(pt_sentence)<=max_len, tf.size(en_sentence)<=max_len)\n",
    "\n",
    "train_datasets = train.map(tf_tokenizer_encoder)\\\n",
    "                      .filter(max_len_filter)\\\n",
    "                      .shuffle(buffer_size=20000)\\\n",
    "                      .padded_batch(batch_size=batch_size, padded_shapes=([-1],[-1]))\n",
    "\n",
    "# for data in train_datasets.take(1):\n",
    "#     print(data)\n",
    "#     print()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3、搭建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:18:02.293929Z",
     "start_time": "2021-09-16T18:18:02.253892Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 2, 3), dtype=float32, numpy=\n",
       "array([[[0.        , 1.        , 0.        ],\n",
       "        [0.84147096, 0.5403023 , 0.00215443]]], dtype=float32)>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def positional_encoding(max_len, d_model):\n",
    "    pos = np.arange(max_len,dtype=float)[:,np.newaxis]\n",
    "    dim = np.arange(d_model,dtype=float)[np.newaxis,:]\n",
    "    matrix = np.multiply(pos, 1 / np.power(10000,2*(dim//2)/np.float32(d_model)))\n",
    "    matrix[:,::2] = np.sin(matrix[:,::2])\n",
    "    matrix[:, 1::2] = np.cos(matrix[:, 1::2])\n",
    "    pos_encoding = np.expand_dims(matrix, 0)\n",
    "    pos_encoding = tf.cast(pos_encoding,tf.float32)\n",
    "    return pos_encoding\n",
    "\n",
    "\n",
    "positional_encoding(2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:18:04.323775Z",
     "start_time": "2021-09-16T18:18:04.084557Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABe50lEQVR4nO2dd3wc1dW/nzOzu9Kq92Jb7p1iY8BguimhBDCQQCAQSEIgjfxCOkne9LwJJG9IJSFASEiDUEIooZlqejHu3Za7bPW62jYz9/fHzK5Xa8la2ZJt2ff5fK5n5k67V5bu3v2ee84RpRQajUajOTwwDnQDNBqNRrP/0IO+RqPRHEboQV+j0WgOI/Sgr9FoNIcRetDXaDSawwg96Gs0Gs1hxJAO+iKySUSWichiEXnPqysRkfkiss7bFg9lGzQajeZAIiL3ikiDiCzv47yIyG9EZL2ILBWRWSnnzhORNd65WwajPftjpj9XKTVTKXWcd3wL8IJSahLwgnes0Wg0hyp/Ac7bw/nzgUleuRH4A4CImMAd3vnpwFUiMn1fG3Mg5J15wH3e/n3AJQegDRqNRrNfUEotAFr2cMk84K/K5S2gSESqgdnAeqVUrVIqBjzgXbtP+Pb1Af2ggOdERAF/VErdBVQqpXYAKKV2iEhFbzeKyI24n3rk5gSPzem2qZk5jUVrtjFz6mi2LlrBmCPGsXhLO7nFhYzq3EFrW5SqY46gqTtGXt1mmjuijJoyijWdPrpbmykfUclI1U7dxkayDaFs6li2rthIjmlQMqWGuliAxvpmlOOQX1rChNIg0a21tDSHsRV0VY8h0tGOUoqsvAIqS3IozQKrcSehhk46LQeAHNMgtyiLSHuULtvBVhAQITfLJLsoiL+4GCc7n86YTWsoRnfYojA/QFG2nxy/gREP44Q6iHV2Ew/FiMUcoo7CVgoHqDnmSAwrgop2Y4fDWOEoVsTCjtrEHAfLIXmtAkbOPALLUURth5itiFkOMcsmZjk4tnKL46Acmyn+Tky/D8Nngs+H+PyI6QfDRBmmu0VwFCxduzXxvwUiiHjbxLFh7Do2DHLyslFK4TgKpQDlbpVKHINy/yGQ7UMEBMF9jCCAIYL3GvecQH1DGyQ8yxMPSvyb7nGuFOPHViV+x5BdPXC74R0ljlev35bxL/sRk2p2/f72cY2knFi2ZkvGzz56yui+H5r2ziWrM3/uzKmjM74WYPGAnj1mAM/dPKB2zJzW+7MXr9qMCjc3KaXKB/TANIyCUQor0u91Kty8Aki98C5vnBsII4GtKcfbvLre6k8Y4LN3Y6gH/ZOVUnXewD5fRFZneqP3g7sL4Nijj1AnLg9x+ysvkn/6V1nw2h18NXcadzx8N6U3PctJH76An774Ix59Yh3feP11/rRoByf+6FP847labvvTTzn9lVIWPvQPrvzul/hp7El+8LG7mZwX4OMP382XjriW44uyueqfv+Y720bxh1/+k3gkxKnXXsHDVx/Jxi9+jH/8fRntcYc3bvgtq154BiceY+xJ5/KVq2Zw7XiTpjv/l7d/t4CXGrsBmFWYzQkXTWLdM7W83txNe9xhRJaPOWMLmXLJ0Yz48IcJTT2TVza388B7W1m6tJ4LTh/HRUdUcUxVDjl1S+h++zm2v7KYune3s3lLB5u647TEbGKO4vbXXyeraR3WukV0rVxG09INNK9porW2je1dMRqjNq1xm7D3gfPDl1+lKWyzuS3MlvYIm5pCbG4OUdfcTagjSnd7lEh3jGhnG49Xv0xuVQnBimJ8JeWYpVWYxRWQW4STlY8TLCJuZtEdd6g582bEMJPF9AcwfAEMnx/DF8DMCmL6Asn9WadMIhyziUYtrJiDFbex4ja25WDFHRzLwbYdbMth9JQyfD6DgM8gJ2AS8BkEfN7WNMjyzgV8Br/5zaMo20Y5uwqA8j7I3H136zg2v7z7FkwBv2lgCJgiGCKYhvuhknp84iW7q4+JZ6Xz2LO3AyQ/nGDXIJ/4Si1ehSEwZu4XMv1zYP4rv8NIGfR7G/8T5ytOvSnj577y2h19nuvtHSUnfz7jZ7/2+u8zvrbopM9lfC3A6308u3DO54gv/vPAPkF6w4rgm3Jxv5fFF/85kiJd7y29/ajVHur3iSEd9JVSdd62QUQexf26Ui8i1d4svxpoGMo2aDQazYARQQxzf71tG1CTcjwKqAMCfdTvE0Om6YtIrojkJ/aBDwDLgceB67zLrgMeG6o2aDQazd4h3rfWPZdB4nHgWm8Vz4lAuyeBvwtMEpFxIhIArvSu3SeGcqZfCTzqfZ31Af9USj0jIu8CD4rI9cAW4PIhbINGo9EMnEGc6YvI/cAZQJmIbAO+B/gBlFJ3Ak8BFwDrgW7gE945S0RuAp4FTOBepdSKfW3PkA36SqlaYEYv9c3AWQN51sqGGL87YwxHfvMV5lxzLW8dfxpXHFXBFW+4n7SPX1TMFz+3iv/53w9y/h/e5vkzwnztuVquOnscL5SeztKnbmX0nAu57dzxvDDlfsK24pxPz+Ht7OmYAqdeP5u1lSfy8F0v0N1cx5iTLuLrZ0/GmX8PSx5fS2PUZlZRNg8uX0081E7J+Bkcc0w1H5hYivPOP9k0fwXL2qPEHEVN0M/4icWMPG0mjz24iva4Q57PYFyun4qjKig77gjU6KPY0hFj4dY2Nm7roKOplaNGzmR0YRZZnTuJ1a6gbe1W2ja20raji8aoTZflEHNcOc8XakI1bceq30JoexPdDV10N4Vpj1h0WQ4h273W9tS/rrhDWyROazhOa3eM5lCM5q4Y0bBFLGwRi1rEI93YsTCB/Bz8uUHM3DyMnHyM7FwkEMTxZaMCOShfFjFLEbN3SYtimIiZ0PYNxDAx/AEMT+s3fAHEMIlZDpblava27RbluIZk5Sgc5W6VUoghmIYQ8BmYhmAa3lbEO95VUvX85O+Z4/T5+2TKLs19T3q+IbtLqn3p+cmfRWa/0gPG6OfB/Z0fKEPVj+GCAGIOzqCvlLqqn/MK6NVYopR6CvdDYdAYakOuRqPRDD9EMPafpr9f0YO+RqPR9MJ+NOTuV/Sgr9FoNOns39U7+xU96Gs0Gk0agmD4/Ae6GUPCsIiyGe1sI3DfY2x773le/KDJo6saOfHtBTx5xz38+EfX8/xpH+X44iBNH/8Jbz/wIPMv/QZlAR+z7v0DN9/xJsq2+c71x9N02808tb2D82sKqP7yD/jGQ0s5d0wRo774Lb715Eq2v/8SueU1XHD2RE7MaWPFXU/ybmuEQr/BrDkjaduyCn9uISOnT+HK42oYGdrI9qdfZO3yRuqjFkFTmF4QYNRJY8k94UzqoxYAlVk+asYWUXXcRLKPmkNroJTFOzp5f3MrLTs6CTVsYXp5HlXZCtm5jsimDbStr6NjWweNUZsOy3W0AggYgtnZ4BlxGwntbKarPkR3S5j2uJM0+NopnqhdMYeWsEVLJE5DR5TmriiRcJxYOE4sarkOUtEwVixMoCAHf0EORm6BV/JxAkEcfxDlyyKuIOYoYo7a5ZhlmkmjbcKIK6lGXNPdxqyE8RbXYOsobNtxPXQdlXTOUo5KGnF9KQbbgOk6YyUcsxL1qfQ05u7umAWewdaQQTd+JsjEMWtfONyNrPsFb6bfXxmO6Jm+RqPR9MJwHdT7Qw/6Go1Gk47IoC3ZPNjQg75Go9GkIRy6M/1hoenXjK7mrE/8H7f/+mvcftz1fO1rp3P8t+dTfczZXF//H/5T28pHH/8Bl/34RXJKR/DY5nau+9oZfH+Jw8bXHufoD17MNSWNPPHrVykJmJz208u5b6NixQuvcvL35vFESwHvzF+EHQsz/oQ5fPHUsbTd/zveen0bXZbDiSVBpn1sLo4Vo3TiLM6aXcPcsYWEFzzKxuc3sLYrhq1gbE6AmllVVM89EWvMsYRtRUnAZGKen+pZ1RTOnEm8+gjWt0Z4b3MrdVvb6WzYQayrlZoCP2brFuKbV9O6divtmztoaQ4nHbMSvlABQ3AathDbsY2u7Y107uiiuzlMS8ymPW4T8fT2xPWmQGs4TnN3jJauGC2hGG0Jx6yoTTxqYYW7sGNhnHiMQEEuZm5+Us9XgVyUPwf82Ti+LCKeY1bM3qXpG552nwi0llqX0PUNQ1ynrBTHLNty9f2Elp/U9h3VQ7NPBFozDemp8Xt1A3HMStBfoLVENM9UBuKY1ZeePxRox6whQAxMX6DfMhzRM32NRqNJRw7dmb4e9DUajSYNQa/T12g0msOKQ3XQHxaafkHrdvIqx3HR0/8LwNLrbmPdS4/y4q0X8Ourf8+1p43m19YsNr/xBF/+yuWcW5mL78u/4q67nqZg1GT+csNsFn3+ayxpjzDvzLGELvgSv7h/CV31m+DD3+Anjyyjef37lE6cxWcvmsbo7W+y5J7XWNUZpSbo58hLpxE4+1pyy2sYd1QNH501kuC6V9nw2Jss3dJOS8ymJGAytSqXmjOmEzhmLus7FAFDqAn6GXFUBVUnTMc3/US2R03e39HBkk0ttNR3EW7dSSzUTpEK4WxdTefaDbStr6djWwc7I4k1+q5AHzCEPJ+BtWMjnVvqCe1sI1QforM9SnvcIeIowvauwGyJe5q6YzR3x5Jr9KNhi2g4TjxqEY9EsGPuGn07Fsafl4vkFGDk5CPBfFQgiPJn4fiDRC0nqecnAq6lB1pLHCf1fG/NvmEaOLaXqctyE6YopbAtp0egNcdROFYsqd8HfGafgdbS1+m72r6T3E/dOil6fOo9gxVoLUFv9/Y87273VuJPv22ofA0Oe/Q6fY1Gozmc0PKORqPRHDaICIZ/eK7O6Q896Gs0Gk06OuCaRqPRHF7oQf8AsrO+i/V3fZRv5P2IO1b8hbKb/8DcG64n9IWPELIdZj39NB+85GdMOOMSbqmuw37oO8z9w9u0bVrOjf9zM6MX3MkPnt/I8cXZHHP79/n4E6vY9MazFI6exk9e2si6VxfgC+Yxc+4MrjmyjA03f4nXalsxRThpSgljrrmCxeF8qo44lo+dOo7pwW4anniU2gVb2BqOEzCEyXkBRp88iuJTz6CtaAKvrWykLGAyviKH6uPGkjtzDuGS8Szf1M4b65po2t5JqHEL0c5WlGPja6qlu3YFrWu30ra5nfrOGK3xXUZcUyDPZ1DgM+jeVuc6ZtW5GbNaYq4DV2p2LXCNuK4hN05jR5SWUJTOUIxoJE4sbBGPWslAa048hmPFkdwCjPwiJLfANeL6slD+HCwMYrbjFUV33E46YaUatgzPaSXViGv6DEyfgW2ppHOW4yhsSyUDryUcsxKOVulB1dIds1IdtHZ3zurbiKtsu4djVl+IgDFAN6VDIdCatgvvwjhEreTDYvWORqPR7E9EBDH6Lxk+6zwRWSMi60Xkll7Of01EFntluYjYIlLindskIsu8c+8NRt+GxUxfo9Fo9jemue9zYhExgTuAc4BtwLsi8rhSamXiGqXUz4Gfe9dfBHxJKdWS8pi5SqmmfW6Mh57pazQaTTrCYM30ZwPrlVK1SqkY8AAwbw/XXwXcPwg96JNhMehXlgZ5ZeLxfOLscZz1rGBmBXn6bLjjgZV89Y6rOP3/3sCKhPjPN8/gmXP/H//KPYX3H32ECWdcwu1nVvDUTfcRcxQXfO0snpcpzH/0dQBmnDOHBx5bSXdzHaOPn8uPPjgd+7Ff8s6/V7EzYjGrKJujPnEK3TMu5O63NnPC8aP44OQy7NceZv0TS1jSHiVsK0Zk+5g0rYyas4+DqSezaGeI51bsZGJegJHHV1M+5xicccewoTXKO5tb2bCpjfb6JiKt9ViRLgBi65fSumozLeubadvRxc6I1UOjD5oGuaZBScCkc2sDXTs6CTWEaI9YtMcdQrazW6A1UzznrK4oDZ1RmhOB1sIWsahFPNKNHQtjR8M4Vgzl2Bh5RRg5+ZCVi+PPQQVycPzZRBNOWY4iZjtELWc3xyzDH0hq/AnnLNPnwzQNxJBkoDXlKBzb0/I9B62EY5ZybJRte5q9kXTM6k3jT5xL0F+gNWXb3s+m/0BrqXp+po5ZqezpD2uwYq/1NubsS2C3Q1PB3jvcKJuDMuiPBLamHG/z6nZ/p0gOcB7wSEq1Ap4TkYUicuPe9aYnWt7RaDSa3dizoT+FsjSt/S6l1F09HrQ7qpc6gIuA19OknZOVUnUiUgHMF5HVSqkFmTSsL/Sgr9FoNOl48k4GNCmljtvD+W1ATcrxKKCuj2uvJE3aUUrVedsGEXkUVy7ap0F/WMg7Go1Gs78ZJHnnXWCSiIwTkQDuwP74bu8SKQROBx5LqcsVkfzEPvABYPm+9mtYDPrhyjG81RIm76+P8cZf7+Px397APSdcz4emlvL0cZ9l0aP3c9VN15B7x1d4YlsH3/zFs2TlF3PXF05m/Rdv4PmGEJceW03hzb/glr8upKV2CaNnn8OvP3Q0OxY9T9HYI/nkJdM5JraWhb96indbI1Rl+zjuvPEUfehT/Ht1E6++uYXrTxxDZcNiNj36PCvWtLAzYlHoNziqJMiYs6aSM+cCNsdzeWFtIxvWNzNmSinVc6YTOPo0dlLA29vaeXNdE8073TX6sVA7AL7sPLrWrqFlbR3tmzvYHrbosJweydDzfK6eX5blo2tbE507uuhqjdASswnZzm6B1kwRgqZBtmEkA611h2LEwnEv2FoMK9zlrtG33DX6djyG4SVQUYGgF2wtmAywFvG23XGb7riNmZI4JRlYzRdIHhu+AIan55um4QZZS0mGbts9A68l1tsrx06uwU8kQ09dl596bIhkHGgtnf4CrQ1EHk+8L/2eoVqjf4guIT9oEAHTJ/2W/lBKWcBNwLPAKuBBpdQKEfmMiHwm5dJLgeeUUqGUukrgNRFZArwD/Fcp9cy+9k3LOxqNRtMLg5XtTCn1FPBUWt2dacd/Af6SVlcLzBiURqSgB32NRqNJQ0QOWY9cPehrNBpNL2TqcTvc0IO+RqPR9MKhOugPC0Pups07+f7LP+O063/LnGuupfQnN7CpO87pbz3LTf/zN0bPuZA7j7P4420vMm9MIQ0rX+eST17GcSsf4P4HVzKjMJuT7vwuNz+xmjUvutm0Pnfl0Uze8iKGL8DRZ83m87NHUfvL/+PlpQ0AnDaphEk3fJSVMoJ7X9jAzhULOaHEpvE/D7DumVrWdkUxBSbnBRg3dwwVZ59FR8V0XtnUwqvL62ncuI2RJ4+n4IRTCVdMYWl9iNfWNdKwrYOOHZuItDfhWDEMX4Cs/GLXMWtdCzvbIjR5AdRs5TpYBU2hwGdQnmWSW5lD544uQvUhWmI27fHejLjuPdmeAbixM0J7V4xId5yoF2gtYcS1o2HsWAQ74ZyVW4DyB72SQ1x8RC2HiJc1KxJ36I47yYBrqaW3QGuGZ8Q1fYbrnGU52JZKGnXTA60lHKiSGbP6CLSWzKaV8neZbsRNJfFcIGm47Y2EY1ZCzs3EMSvdiLunQGuD5ZjVG9oxaxAR9/ekvzIc0TN9jUajSUMQDN+wmBMPGD3oazQaTTpy6IZW1oO+RqPR9MJgLdk82BgW31/8Ofmc80Yphj/Ai+fDL+9+n1vuvJqTf/U+0fYmnv7+OTx1yicwRfjAU79m0txLueu8Kv57/e/pshwu+9Y5zA8ew+P/egXl2Bx7/ql89og8lvzgt4ybcw7/d+mROP/+Ga/dv4w6L9DajBtPJ3zcpfx6QS21768m1LgVe8EDrHlkIe+3RQjbipqgn2lHVTDm/BPgqDN5b0eIJ5fuYMfGFrrqN1F58rGoibNZ3xrl9dpm1ta20rp9Z49Aa4HcQoLFVTStaaS5rvdAawU+k5KASWFJNvnVeXTt6KIlHKcl5niOWT0DrSWSpwRNgzyf0NARJdLtJk6JhuM9Aq3ZsQjKsXHirqZPsAAnK69HoLVISqC1hGNW1HJ6BFpL6vlpgdYMn1vEoN9Aa4k2JJ2zTKPPQGsB03B1VUP6DLSWcMxK1fPdZ2cWaG1vJnqZBlrblz+8Qy3Q2sE4troB1/ovw5Ehb7aImCKySESe9I5LRGS+iKzztsVD3QaNRqMZEJ68018ZjuyPz6ov4rofJ7gFeEEpNQl4wTvWaDSagwjBMI1+y3BkSFstIqOADwL3pFTPA+7z9u8DLhnKNmg0Gs1AET3T32t+BXwdSBVdK5VSOwC8bUVvN4rIjSLynoi8V5kV4c2//5XX7v40t8++katPHMlfp36CJf95gC9+83rkh9fz5I5OPv3dc7l1xwge+NpprPjkx3m+IcSVc8fi++xtfO2ed2mpXcL4k8/jjsuPpvHX3+GplzZz0xVHcVTrQt657UnebQ1TE/Rz4qVTKLj8c9y/vIHXXt9M66blmIEg6+9/hkWrmtkZsSgJmMysymPceUeRfdJFrI9k89TKejasbaZty2oi7Y0EjpnLdjuXN7e28da6JprqOrxk6G64bF92HtnFleRXVNNW28b2sOUlQ+8ZaK08y6Q8x09uRS75owppb4mk6Pm7J0NPJFzJ8xkU+k0ioTiRUIxoOE4sHMYKdxGPdHmB1mLYKVp6aqA1d32+G2Qtaik6o3ZS0++O2xkHWjN97ja5Tn8PgdYSJWD21PF7C7RmegnOYfADrRmSmdbc1zr+PQVaO5j0/APNwdz0wcqRe7AxZIO+iFwINCilFu7N/Uqpu5RSxymljisrLR3k1mk0Gk3fiNC7Q2BaGY4M5ZLNk4GLReQCIBsoEJG/A/UiUq2U2iEi1UDDELZBo9Fo9orhOqj3x5DN9JVS31RKjVJKjcVNHPCiUuoa3AQC13mXXUdK0gCNRqM5GBD6n+UP1w+FA+GcdSvwoIhcD2wBLj8AbdBoNJo+EYGADsOw9yilXgZe9vabgbMGcn/T8jVc/8Bfab78QgAmPfMcF1z0PY668Aq+E3yfW+58l6tPHEnLx3/K7df/ls9d1MkPn1zH3PIcjrvnV1z0j8Wsf+VJSifO4nsfP5ZR7/6dh367gLqIxS1Tc1j52V/w/JpmAoZwxqwqJn7+M7zelc+9z77PjmVvYsfClE0+npUvvMSGUIyAIRxZkMWED0yg/AMX0Fg0kfnL63lj2U6aNm6ku7kO5di0F03g3Y1tPL+ynp2b2+jYUUukvcl1EAoEyS4sI7d8NMWVedR1RGmKWT0CreX5DIr9JuVZJvkj8igYlU/eyHJaYitpj9s9nLhgl1NWItBaod8gGDCJdMeSjlnJbFnxmBtoLe4ac3cZcnNR/hxi4vOCrDlELdXDgNsdtwl5AdcMX8DLoLXLiGv63ABriUBrIm4ck1jU9oKr9Qy05lgxlN3TkNtXoLWAz0gGWvN7Dlp7MuKmO2b1RWqgtUwncOnPyyTQ2sE2jAxkrjrYAcYOaiOugG+YzuT7Q4dh0Gg0mjSEQ1fT14O+RqPRpCPDV7Pvj4Pt26ZGo9EccNyZvtFvyehZIueJyBoRWS8iu0UgEJEzRKRdRBZ75buZ3rs3DIuZvq3g1vaH+NarW/jN8r8w4StPkltewxvfmMOdI2YzLT+LE55+lKO+8yLdzXX8+evzKfabXHzXDfx6Sx5vPPwQgdxCrvjo6XyooIFXbvkzrzeHmVGYTfMffsD8J9fTErO5sDqfY740j601J3Pbw8vY+N77RNobya+ewPhZU3n/0QgxR3FkQRZTThxJzUVnYU0/kwXrWnl84XZ21DbQVb8JOxbGl53HsoZuXlrbSO2GFtrrttPdXIcdCyOGSSC3kNzy0RSV5zJyRD47I7sSp0BPPb+wIpeCUfkUjK4gf3QlLTGbkOeUlR5oLeg5ZeX5DAr8JsHibKJhi2jE1fPtWNjb7kqckigATlYeti+bSNzV8qO2ImI5KXq+Q9RyCMdsV8NPCbJm+AK7kqYkgq2ZktT3Hc8xy7YcHNvBtqxk4pR056xEoLV0pyxTBH/CIzIliUp/iVNSz/cVaE3SNHhjL0KR9eYoNVja9YF0zBquCUP2hcGY6YuICdwBnANsA94VkceVUivTLn1VKXXhXt47IIbFoK/RaDT7E0NksFbvzAbWK6VqAUTkAdxQNJkM3Ptyb59oeUej0Wh6wRTptwBliXAxXrkx7TEjga0px9u8unTmiMgSEXlaRI4Y4L0DQs/0NRqNJo1EGIYMaFJKHbenR/VSp9KO3wfGKKW6vAgG/wEmZXjvgBkWM/2qI8bzrRv+zv/87wc561mhftkCnvjltbx+0jnUReJ8/Mkfcu5fVrPxtcc54cor2NQd59r/dzJLjrmOX/z+BcKt9Rxz0fncdu54ln/9mzy1qomqbB/nXHM0C375Emu7YswqyubYL5wGF9zEr17dxNJXV9GxbS3ZheWMOvoYPnbGeNrjDjVBP0dPLWXSZXMwZl/Euzu6+c/i7WxZ00T7lpVEO1swfAFyykbwSm0zi9c20by9ia76TcRD7YCbOCWndASFlWVUjCxg1phiL9BaInGKUOBz9fzS4mzyRuSRP6qY/NGVBEaOocNyE6ck1ujv0vOFXNNdn1/oN8gqDJBdnE0kFCPWHcKKdBEP7wq0lpq0JIHyB4lYrm4fsd2E6F0xi66YTdjT9buiFl0Ra9f6fG+NvunzuSFnvcQppk96rNd3lLc2PyVxSm/F8dbp7xZwLSVxSmKtfnqkw94Sp6TTX+KUfQm0lvoc6DtxykC1eB1obf8zSB6524CalONRQF3qBUqpDqVUl7f/FOAXkbJM7t0b9Exfo9Fo0hhE56x3gUkiMg7YjhuS5qM93yVVQL1SSonIbNz5QTPQ1t+9e4Me9DUajSYNYXAMuUopS0RuAp4FTOBepdQKEfmMd/5O4MPAZ0XEAsLAlUopBfR67762SQ/6Go1Gk8YANP1+8SSbp9Lq7kzZ/x3wu0zv3Vf0oK/RaDRpHMphGIaFIXdlY5yPnlTDg6d9hTf+eh9f/+EXKPrpDTy4rIEv//iD3No9gzf/8U/GnzaPZz4zm6vPHEvut//A9b9+ncbVbzFp7jz+fN2xNN12M489sQ5bKS44fTTjvvEdFjR1UxP0c/rl0ym7/mvcu3gHTz2/nqa172IGglRMP4GLTh/HpVPLKAmYHFudx6RLjiH3zA+x3irg4SV1LFveQMvGlYRb6xHDJFhcSVHNZF5cvpOGLW101q3vkS0rWDqCgqpRlFbnMWtMMUdVF/TIllXoOWWV5/gpGJVP4egiCsZWk11Tg3/E2IyyZeUUZJFdlE2wOLtHtiw7Fk4GWks34gJEbEXYUkT6yJYVirlG3O6Y3Wu2rF2G292dtFKds5JG23hsNyOucuxeHbNSs2UlHLT8hvQaaC2V9D5mki0r3VlrT8/b9YyegdYGy4i7p3ftDw6nQGtJdBIVjUajOXxIxNM/FNGDvkaj0fSCHvQ1Go3mMME4hJOoDIteRTraKHrov9zy5V8w55pr+XrLw/zqj+/xmcumsOzS7/KLn95HyfgZPPbtuaz75IeY9c/7uOyPb7P+lcepmjGXX336BCrn/5onfv0qdRGLCyaVcMyPbubprgryfAbnnjGaSV//Ok83ZXPPE6uoW7IA5diUTT6eU04Zy3XHjqJk0+scX5zN5IunUTHvcrbnT+DxVfW8vriO+nVrCDVudQOF5ZdQMGoKVWOK2bmpjbYtqwm31icTpwSLK8mvHEPZyAKOGlvCjFGFTCnLwVYJPd+gLGBSle1z9fxRBRSMqyZ39Ej81WNRxSN6TZyyS883yA36CBa7en52cfYuPT8axrHiuyVO6fGzthXRtMQpnbFdiVO6IhbhmOug1VviFJ/fTOr6SSctT+u3bWePiVMcp2cSlVTHLL9h9Eickgi4ltCbM02cknrcV+KUvdHzk/f2ct9g6/kDff++Pe8w1PNBa/oajUZzOCEkY+sccuhBX6PRaHrhUA0nrQd9jUajSUMgmavhUGNYaPqjaqo49RO/YsyJH+DF8+GH193LxRNLKLn7Ea655Z+IYXDHdy4h946vcO9Dq/jksw0s/PejFIyazLc/exqnNbzEc1+6nyXtEc6uyOWU265jxYjT+MGDSzjviHKO/vanWRiYwm2Pr2TTO28QD7VTPPZIps+ZxBdOHc/40DrqHrifqedPYNSHL6GtZjZPr2vmibe3smPtZrp2bsKxYvhzCykYOZmqseWcNL2Cli0bCLfW41gxDF+A7MIy8qrGUVKdz6TRRcwaU8QRFXmMzPP3SIRele2jYGQ+RWMLKRhXRcHYanwjxiFlo7DzK5OJU1KDrCX0/MJsH9melp9TlkN2aWFSz+8rcUoCMUzCcYeIp+d3xWy6YtauQGsRd41+Z9QiHLN66Pk+v5nU7g1Tdmn5KWv2laOwLSup5zt7aEuPdfopwdUMEfxmypp9Qwas56cnTkldV58efK23+/uit0ToPX6+gzRz7Os5B7ueP6xI/L71U4Yjeqav0Wg0aQjgzzAd4nBDD/oajUaTxqEs7+hBX6PRaNKR4Svf9Ice9DUajSYN4dC1aQwL0aqofQfZheUs/f4J3D77RqblZ3HG+y8x91vP0lG3gW9/5+Ocs+Ru/njbi4zI9vP4vf/GH8zjE5+6gBvK6nnlU7fxbH2IWUXZnP2jeTSc/Em++MBi1r7yMif+4Gq2TD6fbz6+gtUL3qS7uY786glMOvFovnzWJGb4Gml66M+sfHAx4z5yIdasi5lf28oDb25m6+rttG1dhRXpwpedR0H1BCrHj2TW9ArmTioj1LgVK9KFGKZrxK0cR9nIEsaPKeKE8SXMqCygJt9PdtuWpBF3ZNBHSWUuRWMKKBxbSeGEkfhHTcSsGoddWE1IsoHUbFm7jLjFAdcpK6csh5zSINml+WSXFmCFu5JGXCfFMas3orYiFLNpj7oG266YTacXZC0RaC0cs5MB13wB/26B1VKzZZk+wTANfD4D27Jco63de7as5LFt7wq01iO4muuglTDiJhy1EuyNU1Z6XXJ/H/7e+wq0lsrePn84G3GH2xjqBvfbcxmO6Jm+RqPRpCHepOJQRA/6Go1Gk8ahLO/oQV+j0Wh6YbjKN/0xLL6/7NjZybJ7ruNfk+YCcM2Sh5n949fZ9u4zXPOlT/L/7Df43Y1/wxThhts/jBUL88GPX8r/Hp/Nm9d9hf+saWZyXoCLvn4W9lX/w02PLGPZ/AWEW3fSdtr1fPu/q1j+0kI6d2wgt7yGiSfO5ubzpjC33KLzP39ixd/f5p26TuTkK3h+Yxt/fXMzG5fvoG3TcuKhdsxAkLyqsVRMGM9R0yr4wNQKjqnOIx5q9/T8cvIqx1FaU0HNmCJOmlTGMdUFjC0KkBuqR21d5TllmZSW51I8vojCcRUUThxJYNR4fCPGYxdW0e3LozlsYwpJLb/AZ1ASMCkJmGQXZxMsC5JTFiRYlk92aSE5FcXYsQhWLLxHPV8MEzFMQjGHztguPb+HU1bEoitq0RmJE47ZmD5fj8BqPn/PoGs+v5FMrBLwGT2SpqQ6ZqXr+YmAa4GU4GoJPd9n7tL1E9o+ZK7nw+4OWH3p+al/8/05ZiV/jhkkTjnY9fyhYLhNmoVdAf32VDJ6lsh5IrJGRNaLyC29nL9aRJZ65Q0RmZFybpOILBORxSLy3mD0Tc/0NRqNJp1BypErIiZwB3AOsA14V0QeV0qtTLlsI3C6UqpVRM4H7gJOSDk/VynVtM+N8dCDvkaj0aThavqD8qjZwHqlVC2AiDwAzAOSg75S6o2U698CRg3Km/tgWMg7Go1Gsz9JhGHorwBlIvJeSrkx7VEjga0px9u8ur64Hng65VgBz4nIwl6evVcMi5l+RXE2b00/kQ2hON9ZeA+n/HUnq559mHM/ewO/n9LA3af/hNa4zc0/OJ91532NU6yV/OXiMSz56FX8681tjMj2c9nn5lB48y/49CPLefOJV+iq30TZ5OP5n2fW8urTC2mpXUKwuIrxJ8zhcx+cyoVjsok88iuW/WUBb65vpS5i8erOOPe9tZm1S3fSWruESHsjhi/g6fmTmTq1jPOOqOT4EfmUddcBkJVfQm55DcUjqxgxuoiTJ5Uxq7qQ8UVZFESakG0riaxfSlW2j6ryHHd9/rgyiifXkD1mAv7Rk7EKRxDOKqap22JnV6xHoLVCv1tySlwtP6csh2BpHsHyYnIqivEXFeFYO3skIE8noecbvgBdMVfLD8fdYGtd0Z56fjjmJlEJRyx8ftPT8k03qFrK+nzDlKSeHwyYBHzGbknQ+9LzlWMn9Xy/2bue70/Z35Oe3xfpidBT6+Dw1vMP28QpqQhkuGKzSSl13J6ftBuqlzpEZC7uoH9KSvXJSqk6EakA5ovIaqXUgoxa1gdDNtMXkWwReUdElojIChH5gVdfIiLzRWSdty0eqjZoNBrN3pBYsjkIhtxtQE3K8Sigbrf3iRwN3APMU0o1J+qVUnXetgF4FFcu2ieGUt6JAmcqpWYAM4HzRORE4BbgBaXUJOAF71ij0WgOIsQL6b3nkgHvApNEZJyIBIArgcd7vElkNPBv4GNKqbUp9bkikp/YBz4ALN/Xng2ZvKOUUkCXd+j3isI1Ypzh1d8HvAx8Y6jaodFoNANlsJyzlFKWiNwEPAuYwL1KqRUi8hnv/J3Ad4FS4PeejGd5klEl8KhX5wP+qZR6Zl/bNKSavrdcaSEwEbhDKfW2iFQqpXYAKKV2eFpVb/feCNwIUJ2TDblD2VKNRqPZhRuGYXCMEUqpp4Cn0uruTNn/FPCpXu6rBWak1+8rQ7p6RyllK6Vm4upYs0XkyAHce5dS6jil1HG54yazoL6Lb714G2c9Kyx86B+cfN3Heewsk7+f+UXWdkX5/FdOp+XjP+WqW1/i8euOZtWN1/GP52opCZh85JPHUP2d3/CV/67h2UcW0LFtLSXjZ3DGB4/j2Sfep2ntu2QXljPuxFO48cJpXDm1COu/v2fZn17ijeWNbA3HyfMZ3PvmJpYurKNp7fuEW3emGHGnMmV6OfNmjOCkmkIqY/VYyxaQlV9CXuVYSmpqGDHWNeIeO7KQiSXZFMVbkW0ria5dRMvyjVSXBikeV0TxpHKKJ48me6xrxLULRxDNKaU5bNEQirG1Pew5ZZmUBFzHrLzibHLKguRW5pJbkU+wvJhgaSGB0hLM4gqsaDjpEJVOqhFXTJP2qOeAFbPpilq0d8d7GHE7IxbRmI0Vt3sYcROZs3wBE8OUpINWIvtVluecleqY1ZcRF0gacVOzZvVmxO1vLXWvhus0I+5uwde8rSGSsRE3lcE24vb5Hm3EHVJE+i/Dkf2yZFMp1YYr45wH1ItINYC3bdgfbdBoNJqBYCD9luHIUK7eKReRIm8/CJwNrMY1YlznXXYd8NhQtUGj0Wj2BuHQnekPpaZfDdzn6foG8KBS6kkReRN4UESuB7YAlw9hGzQajWavGA4xjfaGoVy9sxQ4ppf6ZuCsgTyrdtNOfvjc7Zz/bgVv/PXPzLnmWp6/pIB/HHcVS9oj/L+bT6H75l9z2Y9fZMubT7L2U/fwt0fXUOg3ufrjM6n56V185dnNPHr/y7RtWk7R2CM5/aI5/PSD05j0yz+QlV/CuBNP59MXT+e6o8qwn/gNi+94ljcW17OpO07QFGYUZvHEu9tpXLOQ7ua6pJ5fPnE6k6aXc8nMkZw8uojqeCPO8gU0vf4WeZUzKKmpoWpsEadNKWfOmGKmleVQarVibF9JbO0impduoHnVdorHu3p+ydSxBCdMIjB2KnZxDdHc8qRT1pb2CFvawhT4TAr9u/T83IpcV9P39PycimKyKsowiyswiysy1vNNXyCp53dE4j30/K5IPKnnx6IWVtzJSM8PBkyyfAYBn5mxnq8cO6nn70qgIr3q+f6Uv8z+Aq0l6vrS8w3pqefvDZnq+fs6nmg9f4gZxjP5/shI3hGRyzxnqnYR6RCRThHpGOrGaTQazYFABm+d/kFHpjP9nwEXKaVWDWVjNBqN5mDhcJd36vWAr9FoDicO0TE/40H/PRH5F/Af3PAKACil/j0UjdJoNJoDyaGcLjHTJZsFQDdu7IeLvHLhUDUqHV8wj/MW1/Dqn10j7kuX5vG3Y6/i/bYIX/zKaYS/egcX/fAFNr72OKPnXMh9D68mz2dw9cdnMvpn93Dzs5t5+J8v0VK7hJLxM5g77xR+fvF0Khf+i6z8EsafdCafu/QIPnF0Oc4Tv2HRb5/i1UU72RCKETSFWUXZHH3mWOpXvbebEXf6UZV8aNYoThtTxAirEWfZyzS++gbb31hP6ZixVI0tYu60it2MuNGV79C0eC3Nq7bTvK6V0ikVuxlxI7nlNHRb7OiKsak1zKaWbmobQ5QEDMqzdhlxcytzyasu7NWIaxSWZWzENXz+jI24juUMyIgb8BkZG3GV42RsxE38YWZqxIXMjbgD/ZvXRtxDi8N6yaZS6hND3RCNRqM5mDhUk41kunpnlIg8KiINIlIvIo+IyJBmd9FoNJoDhXjpEvsrw5FMP8z+jOtJOwI368sTXp1Go9Eckhyq8k6mg365UurPSinLK38ByoewXT04clQ+r9/3F+becD0vng/3HHsNyzuifPU7H6Dt//2aD373OTa/8QTjT5vH/bfMpcBncO1nZlNz+9/49BO1PPy352ipXULpxFmc96FTuX3eEZS/cR/vfu9eJp56Fl/88JF8fHoh8Yd/znu3P8HL7+9kU7cbZO344iAzzxnH5I9+IKnn54+YQOWkIzh6RhWXHzuKuWOLGBnbgb1oPg0vv8bWV9dSt6yBkeOLOfuISk4ZW8L08hzK4s0YW5cTWf4WjYvX0bh8G01rmmloCFF6xHhyJk0hMP4IrJIxRHLLaey22N7h6vkbPT1/c1Ooh1NWapC13OrSXXp+aRVSVIETLNzt59mXnm/4Ahnp+VbcDbg2ED0/YBoZ6/lAxnq+aQxMz0+Q0PMNGRw9v8fP9wDq+XvzfK3n747gDo79leFIpu1uEpFrRMT0yjVAc793aTQazTBFRPotw5FMB/1PAlcAO4EdwIe9Oo1Gozn0SPkWuKcyHMl09c4W4OIhbotGo9EcFAgwSDlUDjr2OOiLyNeVUj8Tkd/SSwZ3pdT/G7KWpdCybA0f/eu93FWzlttnf5cOy+abv/wQi879Op+45T/UL1/AtHM/zL++dApVj93KiFvOIvjlX3LlP5aw4OHn6KrfRMX0k7nksuP54Qcmkv3M73jrJ4/w4qomvvnHGVxSYxD6+60s+sOLvLauhbqIRaHf1fOPuGAC4z5yIcacyzB/+j1Pz5/CzKMrucRLmlIe2kJ80YvUv/oOdW9tpG51M+u7Ypx7VBUn1hQxqSRIUbgetiwjvOp9mpZuoHllHc3rW2loCbMzYhOcOBX/mKlYxaPoDhTRFLLY3hFlS3uEjc0hNjd3s7kpRFdbhPzSHHIrc8irzCWnoiCp5wdKSzGKKjCLy5GCMpxgISpN00/V801/wNv3YwaCGP4A7d1x2rrjbuC1SJxwzCYcsTwd39PzYzaOrQjmBTB9Bj6/gWEamJ6Wn0iaEvCZ7rHpHmeq5yvHxpei4ft77LsxTxJ6fqoe3VfCkz3p+dBTz9+1hn/v0Hr+ocNwlW/6o7/f7UTohfdw0x6mF41GoznkcD1yB0feEZHzRGSNiKwXkVt6OS8i8hvv/FIRmZXpvXvDHmf6SqknvN1updRDaQ3VcfA1Gs0hy2DM8718IncA5wDbgHdF5HGl1MqUy84HJnnlBOAPwAkZ3jtgMv0W+80M6zQajeYQwJUQ+ysZMBtYr5SqVUrFgAeAeWnXzAP+qlzeAoq8VLKZ3Dtg+tP0zwcuAEaKyG9SThUA1r6+XKPRaA5KMne+KhOR91KO71JK3ZVyPBLYmnK8DXc2Tz/XjMzw3gHT3+qdOlw9/2J6avidwJf29eWZEnMUv1NP8qNz7qPAZ/KtB7/I/SMu4Zav/42ObWs59vKrefTzJ2L/6svc/fOXuGLz+1x2z7sseuJZIu2NjDz+Aj5x+VF8/ZTRRP72Ixbc9jTPb2mny3K4rDxE892/YdEfX+P17R00Rm3Ks0xOKMlh6mXTqPnwPNTsS3itrpui0dMYMXUis2dUc/GRVcwemU9R81qiC59nx4L32P7WFrbVtrG+K0ZTzOa6saVMKA6Q17EVZ+MSulcspnlFLU0r62mtbWNnW4SdEYvWuI1v3JFYxaPoMvM8p6wom9rCbG7upraxi7qWMF1tEUIdUfJH5JFbkUNORSHBimJyq0rxlyaCrJVDXilOsBAnWIjtz0n+HJMOWYaJ6Q9gpDhlGf4AvkCwhxG3K2IRi9m9GnGtmN3DiBvwDLgBn0FOwOzhlJXl1fdmxN1lyN1lxFWOjSngNw3XYLsHI24ikUWmRlwgYyPuQA15AzHiDnS531AYcTV9I0ohffxOpdGklDpuT4/qpS59UUxf12Ry74DpT9NfAiwRkX8opfTMXqPRHDaIcgbjMduAmpTjUbiT6UyuCWRw74DZo6YvIg96u4s8q3KiLBORpfv6co1Gozk4UaCc/kv/vAtMEpFxIhIArsSNY5bK48C13iqeE4F2pdSODO8dMP3JO1/0tvstdr5Go9EcFKh9VlJQSlkichPwLGAC9yqlVojIZ7zzdwJP4dpO1+PmLfnEnu7d1zb1J+/s8HabgLBSyhGRycBU4Ol9fXmmVB8xjm9d+2dOLAnykVd+z9fWlfGnr9+JY8U479Mf519XTqP2C1fxz/tXuDr9r15j1fNPoRybiadfzNevnslHRysafnYzb/7+NRY0dQNwcmmQrb/4IYv//j6vN4fpshxqgn5mjylg6odmUv2hywlNPp0Xa9t44L2tjJkxlbnHjOCCaZUcW51L9taFhN6az/YFi6l7p46N2zrYGrZoidnEHMW0smyymtZhrVtE5/IlNC/fSPOaJlpr29jeFaMxatMatwnbDlbZeNodP41dFlvaw2xpj7CpKURtYxf1rWFCHVG626N0d0XJr84jWFFEblUJwYpi/GWVGF7SFHKLcLILcbILiJtZdMfspENWoqTr+WZW0Au6FqA9HKMzYhGO2USjFlZsV4A123KSCVRs28HnN/D5TXw9tHyjVz0/qen3oeenOmlBTz3f3adXPd8QGZCeDz31/PQAa3ur5/f2/MQ79nR+MNB6/hCgVKYz+QwepZ7CHdhT6+5M2VfA5zO9d1/JdMnmAiBbREYCL+B+Ev1lMBui0Wg0BxOinH7LcCTTQV+UUt3AZcBvlVKXAtOHrlkajUZzIFHgWP2XYUjGg76IzAGuBv7r1WWaVF2j0WiGF4rBMuQedGQ6cN+M64H7qGeEGA+8NGStSmNVs83/HVPFSS88wVn3ruCtf/6OvKqxfOnmy7hlfBdvnHMBD71TR0nA5JOXT+P3Tz1CdmE50848k9uumsmp1LL2mz/hpYdXs6Q9QqHf4NSyXGZ8cjbP/f51lrRHsZViWn4Wxx1dwZQrZlN80dXsKJzMMysaeOCdrWxa2cCnP3I0500uZ3Kewlz5Ai2vv8j211ayY+FO1jd1UxexaI/b2AoChpBdt5ToqndpW7aS5uWbaFnfSvPmdraHLZpiNu1xV/u3FTRZfhq742xsDbOlPUxtQ4jNzSGaW8N0d0QJdUSJhCLEOlvIG19GTnUJwfLiZMIUs9hNmOJk5aOChUTw0R1zCMWdXUHW/IEeCVMMT9v3BYJJbb+t2w2yFk8kTInZ2LbjafoKK26naPomWT0CrBkEAz4CptGjLuAzMA3BibsJ2vvT8x3HdtfleynpUvX89LX6fdGXng99J0xJ1/P3dS39vq7NzwSt5w8VCpzhOaj3R6ahlV8BXhGRfBHJU0rVAvslwqZGo9EcCIarZt8fmSZGP0pEFgHLgZUislBEjhjapmk0Gs0B5DCXd/4IfFkp9RKAiJwB3A2cNDTN0mg0mgOIUpBZGIZhR6aDfm5iwAdQSr0sIrlD1CaNRqM54Byq8k6mg36tiHwH+Jt3fA2wcWiatDvh9laqF7/HUf/zIhtfe5zRcy7kri+fypzVD/Loib/n+YYQMwqzmffVuRR/9ZcUXvV7Tr3oZG6fdwRVix7i7Z/8hflvbqcuYjEi28cZR1cw48Yzybn4Rt7939MImsLxxUGOOn00k688C//pV7DaLuGRhdt5+t1tbF9bR/uWVVxx5AcYYTXivP0y9a+/xfY317FzSQNrOmPURy26LPeXJGgKZQEfkfdeoGnxWppXbad5XSsNDaFkgLX2uEPMcT3+TIHN7RHXIaulm9rGEJubQnS0RejuiNLdGSUa6iIeaice6SJ/dCVZFYkAaxUYhWXJAGtOVj7dlqI7bhOyHMJxxw2yZpq7GXGTBlwva5YvkEV3xCLmGXEdy0kGW7M9423CiGtbDlkBNzNWVppDVroRN9U5C3bPkpW6dRLOWZ4R12/07pCVOE73odqTATeVwTbipjPURlxtwB1qBs8562BjIInRy4F/e6UMz1VYo9FoDkkOR01fRLKBzwATgWXAV5RS8f3RMI1GozlgDGIYhoON/uSd+4A48CpuSq9puGv2NRqN5pBFOHw1/elKqaMARORPwDtD36TdGTGqipM+/lu6m+s46drr+M8Nx9P6/U9z++/foi4S59JJJZz+u89Te/TlfPQPb3PLl+fx+ZmldNzzHZ6//QVe2tlF2HaYVZTNnPPGM/mGK4mdeDn/XNVEVbaPEypzmXLpEYy64kPYx3yQFzd38OCiDby3eAf169bRUbcBK9JFTftqIu/Op+7VRWx7aytbN7WxMRSnyQuwZgrk+Qwqs3yMDPqoe3URjSvraatto64jys6ITYdl02U52F4AP1MgaBqsbOhiU3M3m5tDbGvqpqs9QrgzRrgrSrSzjVh3O1a4CzsWIbumBrO43AuwVowd9AKs+YJ0xxzCliIUdwjFbNqjVp8JUxIOWa6Dlh/TNIiGLdcRy3Yds1IDrNmWg2M72JaFcmyCAbPPhCmBNMesgGnQV8KUBAk9X9l2nwlT0vV8I0XdHoiev6cAa8mAbHspnGei5+9LQDet5+8PFNiH5uqd/jT9pJQz0CQqIlIjIi+JyCoRWSEiX/TqS0Rkvois87bFe9FujUajGToO4TAM/Q36M0SkwyudwNGJfRHp6OdeC9cGMA04Efi8iEwHbgFeUEpNwo3Yecu+dkKj0WgGm0M1ymZ/8fTNvX2wF4t/h7ffKSKrcBP9zgPO8C67D3gZ+Mbevkej0WgGn8PXkDsoiMhY4BjgbaAykZxFKbVDRCr6uOdG4EaAkYV5+I/M4+e/+iqfLdzMKyedwSPLG6jM8vGFT85kwo9/wb2bfdz+vy+y5Z35vHDu5az6zP/jhSfWs6ozSknA5OzRxRz9iROpvObTrAuO5675G5j/+mbumjOSaVeeTP65H2Fb3gSeWryTB9/awpbVjbTULqW7uQ7l2Piy82h+7B/JAGsbWiNsDceT+nzAEEoCJpVZPkbnBygeX8S2t7bStLWDnZFdAdbC9q5sPAFDyPMZFPgMFm9tZ3NziNbWCKGOCOGuWDLAWqy7HTsaxoqEsOMxfJU1uwKsBQtRWfmElUnYC7AWthzawhZdMcvV9APZfQZYM3wBfH7TS3JueoHWEpr+7mvzlWPjWDGceIz8bP8e1+abhhDwGfgNA1N6avmpWydFi1eejmp6wdV60/Ld3w9XzxfJXMtPXJfJ2vy9kdyHWsvv7R37k31s+vDjEB30M12nv9eISB7wCHCzUqo/SSiJUuoupdRxSqnjSnODQ9dAjUajSScRhqG/MgwZ0kFfRPy4A/4/lFL/9qrrRaTaO18NNAxlGzQajWbgKJQV77fsK5ksbOlrUYx37vsisl1EFnvlgv7eOWSDvrjfY/8ErFJK3Z5y6nHgOm//OuCxoWqDRqPR7BWK/TXTz2RhS1+LYhL8Uik10yv95tMdypn+ycDHgDPTPoVuBc4RkXXAOd6xRqPRHDQoFMq2+y2DwDzcBS1420t2a4tSO5RS73v7nUBiUcxeMWSGXKXUa/RtdzprIM+qq2tn2b2fwv7Vl7n95y+xIRTjolEFnPnbT7D95E9x/r+WsPjZ1+jYtpb86gnMv+hLPL+lnS7L4ciCLE6eO4Zpn/kw6oxreWhNM3/8z2I2LN5M8/r3mfXjG1GzL+GVum4eemkDby2qY+faDXTs2EA81I4YJjmlI8ivnsiqB+5iW20b67tiPRyyCv0GZQHXIauqOo+SScWUTB7Bi3e9lQyw1ptDVsKIWxIwmb+9na62iJshqztGtLODeHc7sVA7diyCFQvjxGM4VgyjfLTrkBUsxPbnEIo7dHsG3M6oTWfMoj1i0RWzaY/GUwKqBT0nLdeIm3DI8vlNDJ+Bz28Qi1o4tkpmzEp3yHLisaQxNxgw+3XI8huCYQh+w51f7MkhK/m749h9GnFTDbiQeSCz1Hdm6pC1LzOiQ80h6/Az4pJp5qwyEXkv5fgupdRdA3hTRgtbEqQtiklwk4hcC7yH+42gdU/P0HluNRqNZjcyjqffpJQ6bk8XiMjzQFUvp749kBb1sSjmD8CPcD+mfgT8AjdAZp/oQV+j0WjSUWpQDLXuo9TZfZ0TkXoRqfZm+X0ubOljUQxKqfqUa+4GnuyvPUO+ZFOj0WiGHyopRe6pDAL9LmzZw6KYxArIBJfiprTdI8Nipl9elM3iWafwn9pWJuQG+PrNJzHqu7dz++JO7v7Oc2xfOB/DF2D8afP42MXT+M/Zd1OeZXLuxFJm3ngaxVfcyEoZwR+eXMOCN7awY+UiQo1bAdgy/WKeWLiTx97ZyuZV9bRtWka4td7VlXMLya8cS9GosVSPK2bRY83UReK0x3clSyn2m1Rl+6gpzKJkUgklE0spnjaGvIkT2fDLBXRZTg+HrKApBM1dWn5JwCS/MIvmnZ2EO2NEQt3EQ+09AqzZnpaf+EWzC6twsvKJOEIoYif1/PaI64zV5Wn6oZhFe3ccfzCvh5afcMjyBUxX0w8YSW0/1BHdzSEr8e6Enp/U9P1mn3q+3zCSQdMSun7qH0pvDlmwS3v3G0avyVISer4hmenMff1hpjtkHaxa/sDfP7jvOuy0/ASJ1TtDz63AgyJyPbAFuBxAREYA9yilLmDXophlIrLYu+9b3kqdn4nITK/Fm4BP9/fCYTHoazQazf5FZWrI3be3KNVMLwtblFJ1wAXefp+LYpRSHxvoO/Wgr9FoNOkoBmtJ5kGHHvQ1Go1mNzJevTPsGBaDvjVqHM+vbue6uWOY/bsf8Lw5nctvf5+1rzxPLNRO+dQTOfmco/jB+VOZ1Pw+D5fncOwVRzL2hk9RP/pkfrl0Bw+/8g6bl6ykfdta7FiY7MJyisYeydceX8GaFQ00rl9JqGErdiyMGQiSUzqCglFTqBpbzMzJZZw6oZTXu6LYCm9tvhdcLcdH6ZhCyqaUUjR5FEVTxuEfOxWpnkBL7Lbk2vyAIQRNIdfcpeUX5/oJluWQV5FDe1N3MrhaQsu3ouEeWn6CaFahtzbfJhxXyXX5CT2/M+pq+V0Rd9+XnYfhdxOgJwKruVq+ic9veGv03Tor3t1jbb5jxVC23aMdyrGxrZiXQCVNz/c0fL/pavKJ9fZ+T9PvT8tP0Nfa/FQNPnW9fjp7MrKJSK/B1Yy0awbKQPT8wU6UrrX8QWYQV+8cbAyLQV+j0Wj2L3qmr9FoNIcP+2/1zn5HD/oajUaThkIl8z8cauhBX6PRaNLRM/0Dy4ZNO/nxf/+X2qMv56z7F7H02Tvpqt9E4ehpHHfZxXz/oumcnFVP/Z1f45l73uSSB24hduLl/GNVE/fe+y61izfRsnEJ8VA7/txCisceyYip4zl55gj+9fcX6ajbgBXpwvAFkgbcijEVTJ5QwulTKjhhVCETirN4nV3B1Ubn+KgYme86ZE0eQfG0MQTGTsUcORm7eBQdRk7S6JsIrlbsNykJGBQHfORW5pBTlkNuRQ45FYWENm3ZZcBNCa7Wm0GyJWwTijuEYjbtUddY2xG1XIOuZ8BtC8fpisTpjtn4gnm9BldLOmf5TUyfYJgG8ajVa3C1pFNWijE3L9vXZ3A1U8BnutuEUbev4GqpJJ2zzN6DqyXqgB6G3d6e0Rf9OWQNhjPVcDXggjbiAq4hNx470K0YEobFoK/RaDT7l/3jnHUg0IO+RqPR9IaWdzQajeYwQanBCqh20DEsBn1fdi7z1k9l4W9/n0yUMvvKj/HtedM5u6iLlr/+gBfueZ3XtrTTGLUJlZ3NH+95j3Xvb6J5/fvEQ+34svMonTiLqskTOH5GNRcfVc2cUfn84Ye/7JEopWx0JZMnlXLG1ApmjyxiQnGAvM7tOIsWMTYnsFuilOJpY8gaNxVzlKvlt5t5NIYttneEyPP1TJRSluUjpyxIbmVuUssPVhSTW1VKdElTv1q+GCaGL0BTt0V7NJ5MlNIVs2gPx2nvjtMZseiKWnRGXG0/FrPJCmb10PKTDlresWEaBDxHKysW7VfLV7a7TSRR6U/LN8XVnjPR8hOYkpmWL3t4Rl9kouXvrfautfxDB716R6PRaA4XlELZetDXaDSawwKlFE7cOtDNGBL0oK/RaDTpKPRM/0ByZE0BL939JwpGTeaka6/juxdN57RgEw1//j7P3/MGr+7ooiVmU55lctGoAj7z8+eS6/J92XmUTT6e6snjmDNzBBcdWcXxI/IobFpN9Jnnk+vyy2vKmTKpNLkuf1xRFrntW3AWLaJr1VKalq7nuEnFyXX5RZNryJowPbkuv83IobHbYltHiC3tYWobQ4zI9vWp5edWlxIsL8ZfWoZZWkUs9Eq/Wr4YJqY/wJb28G7r8jsjFu3hGN0xO6nlx6M2VtwmEPT3uS4/kBI0LSdgYqcFeetNy0+UXL/Zr5bv7rsaPfSv5Sf7LJlp+YbIXhncBlvLT3/OYDyvN7SWv//Qg75Go9EcJiilcHQ8fY1Gozl80Kt3NBqN5nBhP63eEZES4F/AWNwct1copVp7uW4T0AnYgKWUOm4g96eyLzmgNRqN5pAksXqnvzII3AK8oJSaBLzgHffFXKXUzMSAvxf3A8Nkpt+ydDWX3vPHZGasjb/6PI88uJy3WsKEbcXYHD9nTyll6hXHUnnZR6i/+q9kF5ZTOmMuNVNHcvYxI7hwWiVHlWfj3/g2XQ8+z5pXllC3cCfTrr6VoyaWcsakMmaNKGBsgR9//Rriry+kdcVymldspHl1M621bcz63Ck9MmPZRaNotH00dltsbutka3uYjY0hNjeH2NnczS2lwWRmrNzKXM8Rq4RgRTG+4nKM4gp8pVU4OUXYsWd69FkMM1kMfwDDM+YaPj9b2sM9MmN1RTynrIiFFbexYo679Up2rj8lW5bhbn0GwYBJVjLrlWvQtWPhZGashPEW6GHAdY8dsnxmj8xYppG+7xpwE1mwUg2ufRlfE/WmsXuwNXANuAlj5t4aIA12N7r2yKS1d4/t83m9MdB3DIUBF7QRd084+8eQOw84w9u/D3gZ+MZQ3q9n+hqNRpOOt2SzvwKUich7KeXGAb6pUim1A8DbVvTdIp4TkYVp78j0/iTDYqav0Wg0+5XMNf2mNLllN0TkeaCql1PfHkCLTlZK1YlIBTBfRFYrpRYM4P4ketDXaDSaNBSDt3pHKXV2X+dEpF5EqpVSO0SkGmjo4xl13rZBRB4FZgMLgIzuT2VYDPoxR/Hn/AUsvuKr3L5wJxtCMYKmMKMwmxmn1jDlqrn4z7iSdaqUP6/YyfjT5jHpiAo+fOwoThtTxCinGWf5EzT9+XW2v7mOnUsaWN8Voy5i8aPLj2ZaWQ7lqh1j61vEXl5I3bL1NC3fSvO6VpobQ2wPW7TGbc7/8EdxSmqI5lXS2G1R3xxnU1sXm1q6qW0Msa2lm/a2CKGOCOHOGNXHVpFbkU9OVSk5FcVklZVgllZhFldgFJbhBAuxsvNxsguTfU3q+L4AYpqYno5v+AIY/gC+QJDahhBdKVp+NJbQ7x2slH3bcrBth/ziYDLAWqCHlu85ZplufZbPwPI0/VRHLEho+k5yHyDH7zph+T2nLFe7d7V8v2G4urxIUtdPvTeV3uoSzlyG9HTEgl069N5qk5Ly7B71adcN1LFqsHV8zQFEKZzYfgnD8DhwHXCrt30s/QIRyQUMpVSnt/8B4IeZ3p+O1vQ1Go0mHQWO4/RbBoFbgXNEZB1wjneMiIwQkae8ayqB10RkCfAO8F+l1DN7un9PDIuZvkaj0exPFPtnnb5Sqhk4q5f6OuACb78WmDGQ+/eEHvQ1Go0mHdUzl/OhxLAY9Kunj+HbH7mDsK2YkBvgymOrmXbFcZRfdjUN5UfxyIZW7n90CxtWLKFpwwpeuPvzTCsyMde/SedDL7LmtWXseH8n63eEqIvEaYnZ2AoChnCmuZnY2+/TtnwFzSs20rS6mdZN7WwPWzRGLTosh7DtYCtoqJpFY7fFpk3tblC1BndNflNrmK62CN1dMSKhGLHOFmLd7Yw8Y7q7Jt/T8c3icpycIpzsQuzsfGJGgFDcoTtkJQOqpa/JN7OCGL4Api+AGQhi+ANsbg71uSbfthSO5dbZtoNyFDm5gd3W5Af9ZlLHTyY39xnJBCrpa/JTtX0Ax7Hddfp9rMlP1fITx5kGW4NdWn5fOn5funwm9Lcmf7CDpGktfziiDtkwDEOm6YvIvSLSICLLU+pKRGS+iKzztsVD9X6NRqPZazJfpz/sGEpD7l+A89LqBuwyrNFoNPsbpRR2zOq3DEeGbND3HAda0qrn4boK420vGar3azQazd6jPFlzz2U4sr81/R4uw553Wa94rsY3AoyurgSC+6eFGo1GozNn7X+UUncBdwHkjpysLppelAyo1lYzm/m1rTzw8lbWrHiBxvUrCTVsxY6FMQNBxj3zf9S+tpTt7+ygtq6TreFdxltToNBvUpnlY3SOj7U/+XEyoNrW7vhuxltwDb55PuE/qxt7BFQLdUQJdUR7GG+tcBd2LIIVDVN44un4SqtQwQKc7ELiwcJdxtuIQzged7NfRSz8wbyk8dbwBTCzgj2Mt2YgiOlzs141NYd7Nd7atuuQ5dgOtmW5GbBsm4qCMT0csXYZdHsWUwQ7FnZ//n0Yb5P/P7ZNjt/o13ibyHyVMMRmkuVKOTamSEbG270JGJap8ba3TFj78g7NMEKBSgwAhxj7e9AfsMuwRqPR7G8Uan9F2dzv7G+P3ITLMGToMqzRaDT7HQXKUf2W4ciQzfRF5H7cOM9lIrIN+B6ui/CDInI9sAW4fKjer9FoNHuLUmDHtHPWgFBKXdXHqQG5DAOE21oZu2Qh/17XxMPPbmHzqqdo27SM7uY6lGPjzy0kf8QESkZPoGpsEX/50o3UReK0x92vZwFDKM/yMSLbx8i8ACWTiimZWErJtDH883tP0Rq3aY/bhFM0vKApBE2DAp9Bod+kPMvkV69sdIOpdcUId4aIh9p76Ph2PObq6AnHpilziGUXEnGEUFwRDjt0x2O0RyzaoxZdMYuuqEVH1CKrsAzD5wZUS2j6hi+Az2/iC/RMgNLVHsaKuRp+Dy3fe3eqg5VjxSjPz95Nx084Y/kNA7/pavF+Q3CsuPv/14eOn9x3bHL85m4aPtBDxzeEjPT89HNmImlKmo6fKrPvy9fUwdbwYWA6/mAnRdHJUAYZpbSmr9FoNIcTjh70NRqN5jBBL9nUaDSawwcFOMPUUNsfetDXaDSadJTShtwDSdXISmZ/4o4eDljB4kpGHHsulaOLOHpyGadOLOP4kQWMLfDzla9EKfSbTMvPYnSOj9IxhZRMLKZk2miKpozDP3YqUj0Bu2gUq778KOAaewv9BrmmQUnApCRgUpzrJ1iWQ15FDrmVuWxcvI54pIt4qD3pgNXDcJuCGCbbnHzC7XbSAasr5hpwO6MW7d1xuiLuflckTk7pyB4OWK7h1sTnNzBS6kyfUFfbupsDVmo7lGNjJ45tm4qCrB4OWH7DzXblZr1yDbGJaJm2FUv2Id1wm4pybLJ9xm4OWKkGV4MUw26aobE/Jy0z5YbeMmXti9G1p3NX788Z7Eib2nA7vFDaOUuj0WgOI/Sgr9FoNIcT2iNXo9FoDh/2k0duJjlGRGSKiCxOKR0icrN37vsisj3l3AX9vXNYzPQrwo3U+QKMOfEDVI0tYs7kck4eX8pRFblU+yKYO1YRW/0SLU+sZt3qrVxzxpik81XepIkExk7FKRuLVTSSxm6LxpDFptZuNm9sYmyOP+l8lV+YRU5pkLzKXHIq8sipKCanqoRgeQlGcQWt31uyRw0/UQy/m+nqne0dSeer9u44nRHXGasr4u53J7JfxR0KyoqTzlc+v+np+EZS409o8lk+gw3vb+jhfOWkaPnKTs145e5X5GcltXzD8Lbiavip+4bs0vEzyXIVMI0ezlepgdUSma/cfenzGX3h2gRSj3eJ2Puqt/em42sNX5OKYr+t00/kGLlVRG7xjr/Roy1KrQFmAoiICWwHHk255JdKqf/L9IXDYtDXaDSa/YpSOPtn9c483HA14OYYeZm0QT+Ns4ANSqnNe/tCLe9oNBpNGkq5M/3+yiDQI8cI0GeOEY8rgfvT6m4SkaVeitp+U9DqQV+j0Wh6IcPMWWUi8l5KuTH9OSLyvIgs76XMG0h7RCQAXAw8lFL9B2ACrvyzA/hFf88ZFvLO9m1tLFxyI1VGN2bdSqKrnqLlgTU0r9rGhtXNNO7sYmfEpilm0WU5/Gz7i8QLqmnsttgUirOxLczmNd3UNq5hc1OI9raIGzitM8a/zhtPbkU+wYpiguVFBCvLMYvLMYsrMIrKcYKFbsnKx4q8Drj6veEL9NDvE8lPDP+uoGnzVzXQFYnTHbN76PdWzEt+YjvJwGllI/J30+9zAmaP5CcJTf/FrpY+9ftECrfU+uJsf6/6vd8wdkt+0pu9IvV5qQTMXcHQ0vX7vhKgZIrZS8IU2D2o2d5o8f3ds7fy+WDr+JoDiMp4Jt+klDpuz49SZ/d1TkQGkmPkfOB9pVR9yrOT+yJyN/Bkfw3WM32NRqNJx1un318ZBAaSY+Qq0qQd74MiwaXA8v5eOCxm+hqNRrM/Uey3gGu95hgRkRHAPUqpC7zjHOAc4NNp9/9MRGZ6Td7Uy/nd0IO+RqPRpKMUdmzoB32lVDO95BhRStUBF6QcdwOlvVz3sYG+Uw/6Go1Gk4ZS4CgdhuGAUV6QxZqTT+eVxm52Rmxa4zZdlkPM84gzxQ2YluczGJHt5zMvtbG5aTuhjijdHVG6O6NEQ26gtFioHSsSwrFiWNEwR971FaSgDCdYiMrOx84uoCvuEIo7hC2HcNyhvcWiPdpJdmF50mBrZgU9A24AMxD0DLhZKY5VJkvXNOJYDlbczWzlGm5tlFLJTFeJLFcnHD+KgM8g6DeTWa4S2a2SxQuSFg+192qwhV2ZrlKDpZXlBHYz2CaO0wOjOSkB1/aEcmz8hvTpRNVbpquBYKbdN9iZrg60yVXbfA9+bD3oazQazeGBAg7ReGt60NdoNJre0DN9jUajOUxwFEn5+FBjWAz6zujxPLWqhTyfQYHPZEKun5KASX5pDjllQXIrc8mtyCenqpScimLG3vNIMslJIihZOongaMvLZtMesWhvseiKxmiP7qQrJUBaezhOOGbRGbEonzq7h2Zv+qRHwhPD9I59BsGAydK3NiY1+0Q7lGP3GiDtmDFzk5q93xTXcUrAZ7pbt97dj0dCe0xwkl5XEvS7ffaSmaQHSEvq733c3xcBU3po04MZIM1t59AkOOntdh0gTZOOlnc0Go3mMEGhtLyj0Wg0hwvakKvRaDSHGXrQP4Cs21zPov/8TzIQGrnFbhC07ALiviDdcYewpWiLO2yP2cT+/D1Mf4BAbmGvgdDMLHfrC/j50kNLiUdTA6C5QdEcb129bTnJJORHzR7dayC0rNS19Clr7F97+JmUdfS71tWn6uWJdfVHVORjCLuto+9tXb0dDSfvz0R7zwu4avuegqAldPKBJDoJpCymH4xAaKmYaQ8YTIl8qAKjaR3/0EEpvXpHo9FoDhsUevWORqPRHDZoTV+j0WgOM7S8o9FoNIcJrqZ/oFsxNAyLQd8MZHPVzmPp2uRln4q1YMUbvUxUNralvMBmrjF2zlWX4/McpHYZWU2CXlaq1IBmv/7lw0Bq5qldhtf0YGY3fPl010nK2JV9ak+G13Br/W596ctQOr44G3ANlv1ln8o0KFqCHL/Rw7Dau3PSgB4J9DTkprOvNk1zCK2i2uCqyQQ909doNJrDBAXslxQqBwA96Gs0Gk0aCqVX72g0Gs3hgrt6Rw/6B4wjx5Tw1B13ZXz9stt/n/G1P/76hoyvPWd8UcbXwsC09+o8/4CePRASzlmDjW9fPbD2gNbdNQeUQ9iQOzSjQT+IyHkiskZE1ovILQeiDRqNRtMXiZl+f2VfEZHLRWSFiDgictwerut1zBSREhGZLyLrvG1xf+/c74O+iJjAHcD5wHTgKhGZvr/bodFoNHvCVv2XQWA5cBmwoK8L+hkzbwFeUEpNAl7wjvfIgZjpzwbWK6VqlVIx4AFg3gFoh0aj0fSKgxuGob+yryilViml1vRz2Z7GzHnAfd7+fcAl/b1T1H42VojIh4HzlFKf8o4/BpyglLop7bobgRu9wyNxPxEPFcqApgPdiEHmUOuT7s/BT199GqOUKt+XB4vIM97z+yMbiKQc36WUytwAuet9LwNfVUq918u5PsdMEWlTShWlXNuqlNqjxHMgDLm9meh2++TxfnB3AYjIe0qpPvWu4cah1h849Pqk+3PwM5R9UkqdN1jPEpHngapeTn1bKfVYJo/opW6vZ+sHYtDfBtSkHI8C6g5AOzQajWbIUUqdvY+P2NOYWS8i1UqpHSJSDTT097ADoem/C0wSkXEiEgCuBB4/AO3QaDSa4cCexszHgeu8/euAfr857PdBXyllATcBzwKrgAeVUiv6uW3AGtlBzqHWHzj0+qT7c/Az7PskIpeKyDZgDvBfEXnWqx8hIk9Bv2PmrcA5IrIOOMc73vM797chV6PRaDQHjgPinKXRaDSaA4Me9DUajeYw4qAe9IdruAYRuVdEGkRkeUpdn+7SIvJNr49rROTcA9PqvhGRGhF5SURWeS7jX/Tqh2WfRCRbRN4RkSVef37g1Q/L/iQQEVNEFonIk97xcO/PJhFZJiKLReQ9r25Y9+mgQCl1UBbABDYA44EAsASYfqDblWHbTwNmActT6n4G3OLt3wLc5u1P9/qWBYzz+mwe6D6k9acamOXt5wNrvXYPyz7hrnvO8/b9wNvAicO1Pyn9+jLwT+DJ4f4757VzE1CWVjes+3QwlIN5pj9swzUopRYALWnVfblLzwMeUEpFlVIbgfW4fT9oUErtUEq97+134q4gGMkw7ZNy6fIO/V5RDNP+AIjIKOCDwD0p1cO2P3vgUOzTfuVgHvRHAltTjrd5dcOVSqXUDnAHUaDCqx9W/RSRscAxuLPjYdsnTwpZjOvMMl8pNaz7A/wK+Do9Ez4N5/6A+0H8nIgs9MKywPDv0wHnYI6nP6iuxwcxw6afIpIHPALcrJTqkL6D3h/0fVJK2cBMESkCHhWRI/dw+UHdHxG5EGhQSi0UkTMyuaWXuoOmPymcrJSqE5EKYL6IrN7DtcOlTwecg3mmf6iFa6j33KRJc5ceFv0UET/ugP8PpdS/veph3ScApVQb8DJwHsO3PycDF4vIJlwZ9EwR+TvDtz8AKKXqvG0D8CiuXDOs+3QwcDAP+odauIa+3KUfB64UkSwRGQdMAt45AO3rE3Gn9H8CVimlbk85NSz7JCLl3gwfEQkCZwOrGab9UUp9Uyk1Sik1Fvfv5EWl1DUM0/4AiEiuiOQn9oEP4EbaHbZ9Omg40JbkPRXgAtyVIhtwI9Id8DZl2O77gR1AHHcGcj1QipvkYJ23LUm5/tteH9cA5x/o9vfSn1NwvyovBRZ75YLh2ifgaGCR15/lwHe9+mHZn7S+ncGu1TvDtj+4q/aWeGVF4u9/OPfpYCk6DINGo9EcRhzM8o5Go9FoBhk96Gs0Gs1hhB70NRqN5jBCD/oajUZzGKEHfY1GozmM0IO+5oAjIrYXSXGFF/nyyyKy17+bIvKtlP2xqdFONZrDHT3oaw4GwkqpmUqpI3BTvl0AfG8fnvet/i/RaA5P9KCvOahQrsv9jcBN4mKKyM9F5F0RWSoinwYQkTNEZIGIPCoiK0XkThExRORWIOh9c/iH91hTRO72vkk853nhajSHJXrQ1xx0KKVqcX83K3C9mduVUscDxwM3eG724MZi+QpwFDABuEwpdQu7vjlc7V03CbjD+ybRBnxov3VGoznI0IO+5mAlETXxA8C1Xhjkt3Hd8Cd5595Rbr4FGzf0xSl9PGujUmqxt78QGDsUDdZohgMHc2hlzWGKiIwHbNwIigJ8QSn1bNo1Z7B76Ny+YopEU/ZtQMs7msMWPdPXHFSISDlwJ/A75QaGehb4rBfaGRGZ7EVdBJjtRWE1gI8Ar3n18cT1Go2mJ3qmrzkYCHryjR+wgL8BiRDO9+DKMe97IZ4b2ZUi703gVlxNfwFuzHWAu4ClIvI+buRFjUbjoaNsaoYlnrzzVaXUhQe4KRrNsELLOxqNRnMYoWf6Go1GcxihZ/oajUZzGKEHfY1GozmM0IO+RqPRHEboQV+j0WgOI/Sgr9FoNIcR/x8VO1DeO+aocwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 50, 512), dtype=float32, numpy=\n",
       "array([[[ 0.00000000e+00,  1.00000000e+00,  0.00000000e+00, ...,\n",
       "          1.00000000e+00,  0.00000000e+00,  1.00000000e+00],\n",
       "        [ 8.41470957e-01,  5.40302277e-01,  8.21856201e-01, ...,\n",
       "          1.00000000e+00,  1.03663289e-04,  1.00000000e+00],\n",
       "        [ 9.09297407e-01, -4.16146845e-01,  9.36414719e-01, ...,\n",
       "          1.00000000e+00,  2.07326579e-04,  1.00000000e+00],\n",
       "        ...,\n",
       "        [ 1.23573124e-01, -9.92335498e-01,  9.77189839e-01, ...,\n",
       "          9.99987245e-01,  4.87215538e-03,  9.99988139e-01],\n",
       "        [-7.68254638e-01, -6.40144348e-01,  7.31235921e-01, ...,\n",
       "          9.99986708e-01,  4.97581763e-03,  9.99987602e-01],\n",
       "        [-9.53752637e-01,  3.00592542e-01, -1.44026920e-01, ...,\n",
       "          9.99986112e-01,  5.07947942e-03,  9.99987125e-01]]],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def plot_position_embedding(position_embedding):\n",
    "# 绘制位置编码\n",
    "    plt.pcolormesh(position_embedding[0],cmap='RdBu') # 【50*512】\n",
    "    plt.xlabel('Depth')\n",
    "    plt.xlim((0,512))\n",
    "    plt.ylabel('Position')\n",
    "    plt.colorbar()\n",
    "    plt.show()\n",
    "\n",
    "position_embedding = positional_encoding(50,512)\n",
    "plot_position_embedding(position_embedding)\n",
    "position_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:18:08.187292Z",
     "start_time": "2021-09-16T18:18:08.137288Z"
    }
   },
   "outputs": [],
   "source": [
    "def scaled_pot_product_attention(Q,K,V,mask=None):\n",
    "    QK = tf.matmul(Q,K,transpose_b=True) # shape:(batch_size, heads_num, seq_len, seq_len)\n",
    "    dk = tf.cast(tf.shape(K)[-1], tf.float32)\n",
    "    scaled_attention_logits = QK / tf.math.sqrt(dk) # shape:(batch_size, heads_num, seq_len, seq_len)\n",
    "\n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += tf.multiply(mask,-1e9) # shape:(batch_size, heads_num, seq_len, seq_len)\n",
    "\n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) # shape:(batch_size, heads_num, seq_len, seq_len)\n",
    "\n",
    "    attention_output = tf.matmul(attention_weights, V)  # shape:(batch_size, heads_num, seq_len, depth)\n",
    "    return attention_output, attention_weights\n",
    "\n",
    "\n",
    "def FeedForward(dff, d_model):\n",
    "    return tf.keras.Sequential([\n",
    "                tf.keras.layers.Dense(units=dff, activation='relu'),\n",
    "                tf.keras.layers.Dense(d_model),\n",
    "            ])\n",
    "\n",
    "def create_padding_mask(input_data):\n",
    "    padding_mask = tf.cast(tf.math.equal(input_data,0),tf.float32)\n",
    "    padding_mask = padding_mask[:,tf.newaxis,tf.newaxis,:]\n",
    "    return padding_mask\n",
    "\n",
    "def create_look_ahead_mask(input_data):\n",
    "    '''\n",
    "    :param input_data: shape:(batch_size, input_seq_len)  input_seq_len包含start跟end\n",
    "    :return:\n",
    "    '''\n",
    "    seq_len = tf.shape(input_data)[1]\n",
    "    return 1-tf.linalg.band_part(tf.ones((seq_len,seq_len)),-1,0)\n",
    "\n",
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, heads_num):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.WQ = tf.keras.layers.Dense(units=d_model)\n",
    "        self.WK = tf.keras.layers.Dense(units=d_model)\n",
    "        self.WV = tf.keras.layers.Dense(units=d_model)\n",
    "        self.dense = tf.keras.layers.Dense(d_model)\n",
    "        self.depth = d_model // heads_num\n",
    "        self.heads_num = heads_num\n",
    "        self.d_model = d_model\n",
    "\n",
    "    def split_head(self, input):\n",
    "        '''\n",
    "        :param input: shape: (batch_size, seq_len, d_model)\n",
    "        :return: shape: (batch_size, heads_num, seq_len, depth)\n",
    "        '''\n",
    "        batch_size = tf.shape(input)[0]\n",
    "        seq_len = tf.shape(input)[1]\n",
    "        input = tf.reshape(input, (batch_size, seq_len, self.heads_num, self.depth)) # shape:(batch_size, seq_len, heads_num, depth)\n",
    "        return tf.transpose(input,perm=[0,2,1,3]) # shape:(batch_size, heads_num, seq_len, depth)\n",
    "\n",
    "    def call(self,q,k,v,padding_mask):\n",
    "        '''\n",
    "        :param q:  shape:(batch_size, input_seq_len, d_model)\n",
    "        :param k:  shape:(batch_size, input_seq_len, d_model)\n",
    "        :param v:  shape:(batch_size, input_seq_len, d_model)\n",
    "        :return:   shape:(batch_size, input_seq_len, d_model)    shape:(batch_size, heads_num, seq_len, seq_len)\n",
    "        '''\n",
    "        Q = self.WQ(q) # shape:(batch_size, seq_len, d_model)\n",
    "        K = self.WK(k) # shape:(batch_size, seq_len, d_model)\n",
    "        V = self.WV(v) # shape:(batch_size, seq_len, d_model)\n",
    "\n",
    "        Q = self.split_head(Q) # shape:(batch_size, heads_num, seq_len, depth)\n",
    "        K = self.split_head(K) # shape:(batch_size, heads_num, seq_len, depth)\n",
    "        V = self.split_head(V) # shape:(batch_size, heads_num, seq_len, depth)\n",
    "\n",
    "        attention_output, attention_weights = scaled_pot_product_attention(Q,K,V,padding_mask) # shape:(batch_size, heads_num, seq_len, depth)\n",
    "        attention_output = tf.transpose(attention_output, perm=[0,2,1,3]) # shape:(batch_size, seq_len, heads_num, depth)\n",
    "        batch_size = tf.shape(q)[0]\n",
    "        seq_len = tf.shape(q)[1]\n",
    "        concat_attention = tf.reshape(attention_output, (batch_size, seq_len, self.d_model)) # shape:(batch_size, input_seq_len, d_model)\n",
    "        output = self.dense(concat_attention) # shape:(batch_size, input_seq_len, d_model)\n",
    "\n",
    "        return output, attention_weights\n",
    "\n",
    "def positional_encoding(max_len, d_model):\n",
    "    pos = np.arange(max_len,dtype=float)[:,np.newaxis]\n",
    "    dim = np.arange(d_model,dtype=float)[np.newaxis,:]\n",
    "    matrix = np.multiply(pos, 1 / np.power(10000,2*(dim//2)/np.float32(d_model)))\n",
    "    matrix[:,::2] = np.sin(matrix[:,::2])\n",
    "    matrix[:, 1::2] = np.cos(matrix[:, 1::2])\n",
    "    pos_encoding = np.expand_dims(matrix, 0)\n",
    "    pos_encoding = tf.cast(pos_encoding,tf.float32)\n",
    "    return pos_encoding\n",
    "\n",
    "# *******************************************************************************\n",
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, heads_num, dff, rate=0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.mha1 = MultiHeadAttention(d_model, heads_num)\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.layer_norm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.mha2 = MultiHeadAttention(d_model, heads_num)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        self.layer_norm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.ffn = FeedForward(dff, d_model)\n",
    "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
    "        self.layer_norm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "    def call(self, input, encoder_ouput, encoder_decoder_padding_mask, masked_attention_mask, training=False):\n",
    "        '''\n",
    "        :param input:          shape:  (batch_size, target_seq_len, d_model)\n",
    "        :param encoder_ouput:  shape:  (batch_size, input_seq_len, d_model)\n",
    "        :param encoder_decoder_padding_mask:\n",
    "        :param masked_attention_mask:\n",
    "        :return:\n",
    "        '''\n",
    "        q = input\n",
    "        k = input\n",
    "        v = input\n",
    "        attn1_output, attn1_weights = self.mha1(q, k, v, masked_attention_mask)  # shape:(batch_size, target_seq_len, d_model)\n",
    "        attn1_output = self.dropout1(attn1_output,training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        output1 = self.layer_norm1(input + attn1_output) # shape:(batch_size, target_seq_len, d_model)\n",
    "\n",
    "        q = output1        # shape: (batch_size, target_seq_len, d_model)\n",
    "        k = encoder_ouput  # shape: (batch_size, input_seq_len, d_model)\n",
    "        v = encoder_ouput  # shape: (batch_size, input_seq_len, d_model)\n",
    "        attn2_output, attn2_weights = self.mha2(q, k, v, encoder_decoder_padding_mask)  # shape:(batch_size, target_seq_len, d_model)\n",
    "        attn2_output = self.dropout2(attn2_output,training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        output2 = self.layer_norm2(output1 + attn2_output) # shape:(batch_size, target_seq_len, d_model)\n",
    "\n",
    "        ffn_output = self.ffn(output2)  # shape:(batch_size, target_seq_len, d_model)\n",
    "        ffn_output = self.dropout3(ffn_output, training=training)  # shape:(batch_size, target_seq_len, d_model)\n",
    "        output3 = self.layer_norm3(output2 + ffn_output)  # shape:(batch_size, target_seq_len, d_model)\n",
    "\n",
    "        return output3\n",
    "\n",
    "class DecoderModel(tf.keras.layers.Layer):\n",
    "    def __init__(self, target_vocab_size, d_model, max_len, heads_num, dff, layers_num, rate=0.1):\n",
    "        super(DecoderModel, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.max_len = max_len\n",
    "        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
    "        self.decoder_layers = [DecoderLayer(d_model, heads_num, dff, rate) for _ in range(layers_num)]\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, input, encoder_ouput, encoder_decoder_padding_mask, training=False):\n",
    "        '''\n",
    "        :param input:         shape:(batch_size, target_seq_len)  target_seq_len只包含start\n",
    "        :param encoder_ouput: shape:(batch_size, input_seq_len, d_model)\n",
    "        :param training:\n",
    "        :return:\n",
    "        '''\n",
    "        decoder_padding_mask = create_padding_mask(input)\n",
    "        look_ahead_mask = create_look_ahead_mask(input)\n",
    "        masked_attention_mask = tf.maximum(decoder_padding_mask, look_ahead_mask)\n",
    "\n",
    "        input_embedding = self.embedding(input) # shape:(batch_size, target_seq_len, d_model)\n",
    "        input_embedding = input_embedding * tf.math.sqrt(tf.cast(self.d_model,tf.float32)) # shape:(batch_size, target_seq_len, d_model)\n",
    "        pos_encoding = positional_encoding(self.max_len, self.d_model)  # shape: (1, max_len, d_model)\n",
    "\n",
    "        input_seq_len = tf.shape(input)[1]\n",
    "        input_pos_embedding = input_embedding + pos_encoding[:,:input_seq_len,:] # shape:(batch_size, input_seq_len, d_model)\n",
    "        x = self.dropout(input_pos_embedding, training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "\n",
    "        for decoder_layer in self.decoder_layers:\n",
    "            x = decoder_layer(x, encoder_ouput, encoder_decoder_padding_mask, masked_attention_mask, training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        decoder_ouput = x\n",
    "        return decoder_ouput # shape:(batch_size, input_seq_len, d_model)\n",
    "# *******************************************************************************\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# *******************************************************************************\n",
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, heads_num, dff, rate=0.1):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.mha = MultiHeadAttention(d_model, heads_num)\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.layer_norm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.ffn = FeedForward(dff, d_model)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        self.layer_norm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "    def call(self, input, padding_mask, training=False):\n",
    "        '''\n",
    "        :param input:  shape: (batch_size, input_seq_len, d_model)\n",
    "        :param training:\n",
    "        :return:\n",
    "        '''\n",
    "        q = input\n",
    "        k = input\n",
    "        v = input\n",
    "        attn_output, attn_weights = self.mha(q,k,v,padding_mask) # shape:(batch_size, input_seq_len, d_model)\n",
    "        attn_output = self.dropout1(attn_output,training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        output1 = self.layer_norm1(input + attn_output) # shape:(batch_size, input_seq_len, d_model)\n",
    "\n",
    "        ffn_output = self.ffn(output1) # shape:(batch_size, input_seq_len, d_model)\n",
    "        ffn_output = self.dropout2(ffn_output,training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        output2 = self.layer_norm2(output1 + ffn_output) # shape:(batch_size, input_seq_len, d_model)\n",
    "\n",
    "        return output2 # shape:(batch_size, input_seq_len, d_model)\n",
    "\n",
    "class EncoderModel(tf.keras.layers.Layer):\n",
    "    def __init__(self, input_vocab_size, d_model, max_len, heads_num, dff, layers_num, rate=0.1):\n",
    "        super(EncoderModel, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.max_len = max_len\n",
    "        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
    "        self.encoder_layers = [EncoderLayer(d_model, heads_num, dff, rate) for _ in range(layers_num)]\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, input, training=False):\n",
    "        '''\n",
    "        :param input:  shape:(batch_size, input_seq_len)  input_seq_len包含start跟end\n",
    "        :param training:\n",
    "        :return:\n",
    "        '''\n",
    "        padding_mask = create_padding_mask(input)\n",
    "\n",
    "        input_embedding = self.embedding(input) # shape:(batch_size, input_seq_len, d_model)\n",
    "        input_embedding = input_embedding * tf.math.sqrt(tf.cast(self.d_model,tf.float32)) # shape:(batch_size, input_seq_len, d_model)\n",
    "        pos_encoding = positional_encoding(self.max_len, self.d_model)  # shape: (1, max_len, d_model)\n",
    "\n",
    "        input_seq_len = tf.shape(input)[1]\n",
    "        input_pos_embedding = input_embedding + pos_encoding[:,:input_seq_len,:] # shape:(batch_size, input_seq_len, d_model)\n",
    "        x = self.dropout(input_pos_embedding, training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        for encoder_layer in self.encoder_layers:\n",
    "            x = encoder_layer(x, padding_mask, training=training) # shape:(batch_size, input_seq_len, d_model)\n",
    "        encoder_ouput = x\n",
    "        return encoder_ouput, padding_mask # shape:(batch_size, input_seq_len, d_model)\n",
    "\n",
    "# *******************************************************************************\n",
    "\n",
    "class Transformer(tf.keras.Model):\n",
    "    def __init__(self, input_vocab_size, target_vocab_size, d_model, max_len, heads_num, dff, layers_num, rate=0.1):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.EncoderModel = EncoderModel(input_vocab_size, d_model, max_len, heads_num, dff, layers_num)\n",
    "        self.DecoderModel = DecoderModel(target_vocab_size, d_model, max_len, heads_num, dff, layers_num)\n",
    "        self.linear = tf.keras.layers.Dense(units=target_vocab_size)\n",
    "\n",
    "\n",
    "    def call(self, encoder_input, decoder_input, training=False):\n",
    "        '''\n",
    "        :param encoder_input:  shape:(batch_size, input_seq_len)\n",
    "        :param decoder_input:  shape:(batch_size, target_seq_len)\n",
    "        :param training:\n",
    "        :return:\n",
    "        '''\n",
    "        encoder_ouput, encoder_decoder_padding_mask = self.EncoderModel(encoder_input, training=training)\n",
    "        decoder_ouput = self.DecoderModel(decoder_input, encoder_ouput, encoder_decoder_padding_mask, training=training)\n",
    "        predictions = self.linear(decoder_ouput) # shape:(batch_size, input_seq_len, d_model)\n",
    "        predictions = tf.nn.softmax(predictions, axis=-1)\n",
    "        return predictions # shape: (batch_size, target_seq_len, target_vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4、模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T23:15:24.700500Z",
     "start_time": "2021-09-16T18:28:41.738725Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 4.987122535705566 Accuracy 0.0\n",
      "Epoch 1 Batch 100 Loss 4.51073694229126 Accuracy 0.026545636355876923\n",
      "Epoch 1 Batch 200 Loss 4.396350383758545 Accuracy 0.03209593892097473\n",
      "Epoch 1 Batch 300 Loss 4.210732460021973 Accuracy 0.033920254558324814\n",
      "Epoch 1 Batch 400 Loss 4.012307167053223 Accuracy 0.03699677065014839\n",
      "Epoch 1 Batch 500 Loss 3.8417892456054688 Accuracy 0.04503483325242996\n",
      "Epoch 1  Loss 3.8383219242095947 Accuracy 0.04515818879008293\n",
      "Time take for 1 epoch:162.11100721359253 secs\n",
      "\n",
      "Epoch 2 Batch 0 Loss 3.2450525760650635 Accuracy 0.0924479141831398\n",
      "Epoch 2 Batch 100 Loss 3.040440082550049 Accuracy 0.09694615751504898\n",
      "Epoch 2 Batch 200 Loss 2.933562755584717 Accuracy 0.10538532584905624\n",
      "Epoch 2 Batch 300 Loss 2.8364346027374268 Accuracy 0.11202184855937958\n",
      "Epoch 2 Batch 400 Loss 2.7677245140075684 Accuracy 0.11792724579572678\n",
      "Epoch 2 Batch 500 Loss 2.6981916427612305 Accuracy 0.12294022738933563\n",
      "Epoch 2  Loss 2.698106288909912 Accuracy 0.12310423702001572\n",
      "Time take for 1 epoch:146.55532932281494 secs\n",
      "\n",
      "Epoch 3 Batch 0 Loss 2.2795474529266357 Accuracy 0.1743749976158142\n",
      "Epoch 3 Batch 100 Loss 2.417585849761963 Accuracy 0.15579009056091309\n",
      "Epoch 3 Batch 200 Loss 2.3739748001098633 Accuracy 0.15692730247974396\n",
      "Epoch 3 Batch 300 Loss 2.346208095550537 Accuracy 0.15984703600406647\n",
      "Epoch 3 Batch 400 Loss 2.3146162033081055 Accuracy 0.1606486290693283\n",
      "Epoch 3 Batch 500 Loss 2.2945375442504883 Accuracy 0.1627989113330841\n",
      "Epoch 3  Loss 2.2947731018066406 Accuracy 0.1628562957048416\n",
      "Time take for 1 epoch:146.41245436668396 secs\n",
      "\n",
      "Epoch 4 Batch 0 Loss 2.3452603816986084 Accuracy 0.18359375\n",
      "Epoch 4 Batch 100 Loss 2.1576595306396484 Accuracy 0.17969045042991638\n",
      "Epoch 4 Batch 200 Loss 2.1490750312805176 Accuracy 0.18072877824306488\n",
      "Epoch 4 Batch 300 Loss 2.11667537689209 Accuracy 0.18167458474636078\n",
      "Epoch 4 Batch 400 Loss 2.102036476135254 Accuracy 0.18304157257080078\n",
      "Epoch 4 Batch 500 Loss 2.0887370109558105 Accuracy 0.18449054658412933\n",
      "Epoch 4  Loss 2.087904691696167 Accuracy 0.1844826489686966\n",
      "Time take for 1 epoch:146.26818990707397 secs\n",
      "\n",
      "Epoch 5 Batch 0 Loss 2.157191753387451 Accuracy 0.21937499940395355\n",
      "Epoch 5 Batch 100 Loss 1.9450966119766235 Accuracy 0.20170560479164124\n",
      "Epoch 5 Batch 200 Loss 1.9368369579315186 Accuracy 0.2014198750257492\n",
      "Epoch 5 Batch 300 Loss 1.9149494171142578 Accuracy 0.20180945098400116\n",
      "Epoch 5 Batch 400 Loss 1.9083302021026611 Accuracy 0.20388834178447723\n",
      "Epoch 5 Batch 500 Loss 1.8969837427139282 Accuracy 0.20527075231075287\n",
      "Epoch 5  Loss 1.8976058959960938 Accuracy 0.20537148416042328\n",
      "Time take for 1 epoch:145.3761088848114 secs\n",
      "\n",
      "Epoch 6 Batch 0 Loss 1.8263124227523804 Accuracy 0.234375\n",
      "Epoch 6 Batch 100 Loss 1.7517502307891846 Accuracy 0.22178180515766144\n",
      "Epoch 6 Batch 200 Loss 1.7317222356796265 Accuracy 0.2210080474615097\n",
      "Epoch 6 Batch 300 Loss 1.7247297763824463 Accuracy 0.22149749100208282\n",
      "Epoch 6 Batch 400 Loss 1.7108535766601562 Accuracy 0.2209543138742447\n",
      "Epoch 6 Batch 500 Loss 1.7178875207901 Accuracy 0.22269919514656067\n",
      "Epoch 6  Loss 1.7187069654464722 Accuracy 0.22285698354244232\n",
      "Time take for 1 epoch:142.9407205581665 secs\n",
      "\n",
      "Epoch 7 Batch 0 Loss 1.5565224885940552 Accuracy 0.2397836595773697\n",
      "Epoch 7 Batch 100 Loss 1.5729258060455322 Accuracy 0.2376844435930252\n",
      "Epoch 7 Batch 200 Loss 1.577652096748352 Accuracy 0.238955557346344\n",
      "Epoch 7 Batch 300 Loss 1.5809427499771118 Accuracy 0.23955585062503815\n",
      "Epoch 7 Batch 400 Loss 1.5698567628860474 Accuracy 0.23949530720710754\n",
      "Epoch 7 Batch 500 Loss 1.565126895904541 Accuracy 0.23913688957691193\n",
      "Epoch 7  Loss 1.5651161670684814 Accuracy 0.2391214668750763\n",
      "Time take for 1 epoch:142.80168104171753 secs\n",
      "\n",
      "Epoch 8 Batch 0 Loss 1.482948660850525 Accuracy 0.25883153080940247\n",
      "Epoch 8 Batch 100 Loss 1.3920866250991821 Accuracy 0.25246649980545044\n",
      "Epoch 8 Batch 200 Loss 1.4068493843078613 Accuracy 0.2537842392921448\n",
      "Epoch 8 Batch 300 Loss 1.4138190746307373 Accuracy 0.2533956468105316\n",
      "Epoch 8 Batch 400 Loss 1.4162293672561646 Accuracy 0.2531524896621704\n",
      "Epoch 8 Batch 500 Loss 1.4200562238693237 Accuracy 0.2525063455104828\n",
      "Epoch 8  Loss 1.4211980104446411 Accuracy 0.2526349127292633\n",
      "Time take for 1 epoch:142.85759973526 secs\n",
      "\n",
      "Epoch 9 Batch 0 Loss 1.1600364446640015 Accuracy 0.2877604067325592\n",
      "Epoch 9 Batch 100 Loss 1.2743676900863647 Accuracy 0.2687186002731323\n",
      "Epoch 9 Batch 200 Loss 1.2830736637115479 Accuracy 0.26876693964004517\n",
      "Epoch 9 Batch 300 Loss 1.2735052108764648 Accuracy 0.2669965326786041\n",
      "Epoch 9 Batch 400 Loss 1.2833688259124756 Accuracy 0.26833635568618774\n",
      "Epoch 9 Batch 500 Loss 1.2856314182281494 Accuracy 0.268801212310791\n",
      "Epoch 9  Loss 1.2853621244430542 Accuracy 0.2686351239681244\n",
      "Time take for 1 epoch:142.85197639465332 secs\n",
      "\n",
      "Epoch 10 Batch 0 Loss 0.9699628949165344 Accuracy 0.24062499403953552\n",
      "Epoch 10 Batch 100 Loss 1.130129337310791 Accuracy 0.29233211278915405\n",
      "Epoch 10 Batch 200 Loss 1.1206086874008179 Accuracy 0.2865152060985565\n",
      "Epoch 10 Batch 300 Loss 1.1311616897583008 Accuracy 0.28664061427116394\n",
      "Epoch 10 Batch 400 Loss 1.1346954107284546 Accuracy 0.28599902987480164\n",
      "Epoch 10 Batch 500 Loss 1.1408754587173462 Accuracy 0.2865229845046997\n",
      "Epoch 10  Loss 1.1403218507766724 Accuracy 0.28648969531059265\n",
      "Time take for 1 epoch:142.8211739063263 secs\n",
      "\n",
      "Epoch 11 Batch 0 Loss 1.0083203315734863 Accuracy 0.33152174949645996\n",
      "Epoch 11 Batch 100 Loss 0.9940115809440613 Accuracy 0.30838754773139954\n",
      "Epoch 11 Batch 200 Loss 0.9865045547485352 Accuracy 0.3030375838279724\n",
      "Epoch 11 Batch 300 Loss 1.0012857913970947 Accuracy 0.3023916482925415\n",
      "Epoch 11 Batch 400 Loss 1.016853928565979 Accuracy 0.30228185653686523\n",
      "Epoch 11 Batch 500 Loss 1.0237077474594116 Accuracy 0.301449716091156\n",
      "Epoch 11  Loss 1.023726463317871 Accuracy 0.30143994092941284\n",
      "Time take for 1 epoch:142.90831112861633 secs\n",
      "\n",
      "Epoch 12 Batch 0 Loss 1.149263858795166 Accuracy 0.3779761791229248\n",
      "Epoch 12 Batch 100 Loss 0.8922573924064636 Accuracy 0.32436567544937134\n",
      "Epoch 12 Batch 200 Loss 0.9127252101898193 Accuracy 0.3226974904537201\n",
      "Epoch 12 Batch 300 Loss 0.91981440782547 Accuracy 0.31850466132164\n",
      "Epoch 12 Batch 400 Loss 0.9263424873352051 Accuracy 0.31706011295318604\n",
      "Epoch 12 Batch 500 Loss 0.9322683811187744 Accuracy 0.3151763379573822\n",
      "Epoch 12  Loss 0.9324678182601929 Accuracy 0.31520652770996094\n",
      "Time take for 1 epoch:142.7065074443817 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-1     ************************\n",
      "\n",
      "Epoch 13 Batch 0 Loss 0.7903605103492737 Accuracy 0.3135080635547638\n",
      "Epoch 13 Batch 100 Loss 0.8093773722648621 Accuracy 0.33644744753837585\n",
      "Epoch 13 Batch 200 Loss 0.8212465643882751 Accuracy 0.3343222737312317\n",
      "Epoch 13 Batch 300 Loss 0.8305183053016663 Accuracy 0.331076979637146\n",
      "Epoch 13 Batch 400 Loss 0.8445708751678467 Accuracy 0.32936546206474304\n",
      "Epoch 13 Batch 500 Loss 0.8553946018218994 Accuracy 0.3278687000274658\n",
      "Epoch 13  Loss 0.8556682467460632 Accuracy 0.3279915153980255\n",
      "Time take for 1 epoch:142.58868288993835 secs\n",
      "\n",
      "Epoch 14 Batch 0 Loss 0.7988878488540649 Accuracy 0.3518750071525574\n",
      "Epoch 14 Batch 100 Loss 0.7477752566337585 Accuracy 0.3490852117538452\n",
      "Epoch 14 Batch 200 Loss 0.7531096339225769 Accuracy 0.34341612458229065\n",
      "Epoch 14 Batch 300 Loss 0.7580415606498718 Accuracy 0.34004276990890503\n",
      "Epoch 14 Batch 400 Loss 0.7698169946670532 Accuracy 0.3382466733455658\n",
      "Epoch 14 Batch 500 Loss 0.7798940539360046 Accuracy 0.3360923230648041\n",
      "Epoch 14  Loss 0.7798829078674316 Accuracy 0.3360738754272461\n",
      "Time take for 1 epoch:142.91367316246033 secs\n",
      "\n",
      "Epoch 15 Batch 0 Loss 0.6090342402458191 Accuracy 0.297821968793869\n",
      "Epoch 15 Batch 100 Loss 0.6747622489929199 Accuracy 0.3526645302772522\n",
      "Epoch 15 Batch 200 Loss 0.677107572555542 Accuracy 0.3478515148162842\n",
      "Epoch 15 Batch 300 Loss 0.6952981948852539 Accuracy 0.3462408781051636\n",
      "Epoch 15 Batch 400 Loss 0.7120851278305054 Accuracy 0.34645015001296997\n",
      "Epoch 15 Batch 500 Loss 0.7245943546295166 Accuracy 0.34542572498321533\n",
      "Epoch 15  Loss 0.7248170375823975 Accuracy 0.34546181559562683\n",
      "Time take for 1 epoch:142.77840518951416 secs\n",
      "\n",
      "Epoch 16 Batch 0 Loss 0.550836443901062 Accuracy 0.3119959533214569\n",
      "Epoch 16 Batch 100 Loss 0.6086200475692749 Accuracy 0.3525330424308777\n",
      "Epoch 16 Batch 200 Loss 0.6282950043678284 Accuracy 0.3551468849182129\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16 Batch 300 Loss 0.6481584310531616 Accuracy 0.3557683229446411\n",
      "Epoch 16 Batch 400 Loss 0.6631523966789246 Accuracy 0.354754775762558\n",
      "Epoch 16 Batch 500 Loss 0.6715425848960876 Accuracy 0.3522303104400635\n",
      "Epoch 16  Loss 0.6718152761459351 Accuracy 0.3522082567214966\n",
      "Time take for 1 epoch:143.09844326972961 secs\n",
      "\n",
      "Epoch 17 Batch 0 Loss 0.6081269383430481 Accuracy 0.408203125\n",
      "Epoch 17 Batch 100 Loss 0.588765561580658 Accuracy 0.3715556263923645\n",
      "Epoch 17 Batch 200 Loss 0.6018527746200562 Accuracy 0.3685212731361389\n",
      "Epoch 17 Batch 300 Loss 0.6077414751052856 Accuracy 0.36440300941467285\n",
      "Epoch 17 Batch 400 Loss 0.6215577125549316 Accuracy 0.36273911595344543\n",
      "Epoch 17 Batch 500 Loss 0.6298189163208008 Accuracy 0.36022648215293884\n",
      "Epoch 17  Loss 0.6302928328514099 Accuracy 0.3603159487247467\n",
      "Time take for 1 epoch:142.8256549835205 secs\n",
      "\n",
      "Epoch 18 Batch 0 Loss 0.5400938391685486 Accuracy 0.35576921701431274\n",
      "Epoch 18 Batch 100 Loss 0.542340099811554 Accuracy 0.3721606731414795\n",
      "Epoch 18 Batch 200 Loss 0.5565290451049805 Accuracy 0.3698497712612152\n",
      "Epoch 18 Batch 300 Loss 0.5719110369682312 Accuracy 0.37157049775123596\n",
      "Epoch 18 Batch 400 Loss 0.5803069472312927 Accuracy 0.36931338906288147\n",
      "Epoch 18 Batch 500 Loss 0.5891420841217041 Accuracy 0.36641693115234375\n",
      "Epoch 18  Loss 0.5895088911056519 Accuracy 0.36649301648139954\n",
      "Time take for 1 epoch:142.88277578353882 secs\n",
      "\n",
      "Epoch 19 Batch 0 Loss 0.4905323088169098 Accuracy 0.3489583432674408\n",
      "Epoch 19 Batch 100 Loss 0.5221209526062012 Accuracy 0.3843280076980591\n",
      "Epoch 19 Batch 200 Loss 0.5262154340744019 Accuracy 0.37723714113235474\n",
      "Epoch 19 Batch 300 Loss 0.5359677672386169 Accuracy 0.37868738174438477\n",
      "Epoch 19 Batch 400 Loss 0.5460456013679504 Accuracy 0.37427085638046265\n",
      "Epoch 19 Batch 500 Loss 0.5564734935760498 Accuracy 0.3723675608634949\n",
      "Epoch 19  Loss 0.5569242835044861 Accuracy 0.3724082410335541\n",
      "Time take for 1 epoch:142.7751920223236 secs\n",
      "\n",
      "Epoch 20 Batch 0 Loss 0.43413543701171875 Accuracy 0.36510416865348816\n",
      "Epoch 20 Batch 100 Loss 0.48089420795440674 Accuracy 0.3791109323501587\n",
      "Epoch 20 Batch 200 Loss 0.4904062747955322 Accuracy 0.3797585666179657\n",
      "Epoch 20 Batch 300 Loss 0.49856531620025635 Accuracy 0.37902510166168213\n",
      "Epoch 20 Batch 400 Loss 0.5101772546768188 Accuracy 0.3778669536113739\n",
      "Epoch 20 Batch 500 Loss 0.5213385820388794 Accuracy 0.3763790428638458\n",
      "Epoch 20  Loss 0.5214787721633911 Accuracy 0.37605780363082886\n",
      "Time take for 1 epoch:143.19037079811096 secs\n",
      "\n",
      "Epoch 21 Batch 0 Loss 0.42167872190475464 Accuracy 0.3660714328289032\n",
      "Epoch 21 Batch 100 Loss 0.45484060049057007 Accuracy 0.39526551961898804\n",
      "Epoch 21 Batch 200 Loss 0.4667416214942932 Accuracy 0.3889516294002533\n",
      "Epoch 21 Batch 300 Loss 0.47727611660957336 Accuracy 0.3865455389022827\n",
      "Epoch 21 Batch 400 Loss 0.4863449037075043 Accuracy 0.38345906138420105\n",
      "Epoch 21 Batch 500 Loss 0.497182160615921 Accuracy 0.38205695152282715\n",
      "Epoch 21  Loss 0.49727413058280945 Accuracy 0.38192087411880493\n",
      "Time take for 1 epoch:143.016521692276 secs\n",
      "\n",
      "Epoch 22 Batch 0 Loss 0.4594951868057251 Accuracy 0.40335649251937866\n",
      "Epoch 22 Batch 100 Loss 0.4407660961151123 Accuracy 0.39269986748695374\n",
      "Epoch 22 Batch 200 Loss 0.4488155245780945 Accuracy 0.3910742402076721\n",
      "Epoch 22 Batch 300 Loss 0.4584354758262634 Accuracy 0.3911583423614502\n",
      "Epoch 22 Batch 400 Loss 0.46482935547828674 Accuracy 0.3896644413471222\n",
      "Epoch 22 Batch 500 Loss 0.47437307238578796 Accuracy 0.3871454894542694\n",
      "Epoch 22  Loss 0.4748024344444275 Accuracy 0.38720181584358215\n",
      "Time take for 1 epoch:142.88217663764954 secs\n",
      "\n",
      "Epoch 23 Batch 0 Loss 0.5485872626304626 Accuracy 0.4309895932674408\n",
      "Epoch 23 Batch 100 Loss 0.4174213707447052 Accuracy 0.40054747462272644\n",
      "Epoch 23 Batch 200 Loss 0.4215070307254791 Accuracy 0.3945782482624054\n",
      "Epoch 23 Batch 300 Loss 0.43295520544052124 Accuracy 0.39475011825561523\n",
      "Epoch 23 Batch 400 Loss 0.4416331648826599 Accuracy 0.3919864296913147\n",
      "Epoch 23 Batch 500 Loss 0.4501989781856537 Accuracy 0.39048609137535095\n",
      "Epoch 23  Loss 0.45037224888801575 Accuracy 0.39050939679145813\n",
      "Time take for 1 epoch:143.20052194595337 secs\n",
      "\n",
      "Epoch 24 Batch 0 Loss 0.3167659640312195 Accuracy 0.34716796875\n",
      "Epoch 24 Batch 100 Loss 0.4022391438484192 Accuracy 0.40330970287323\n",
      "Epoch 24 Batch 200 Loss 0.40625426173210144 Accuracy 0.39776745438575745\n",
      "Epoch 24 Batch 300 Loss 0.4173298180103302 Accuracy 0.39851754903793335\n",
      "Epoch 24 Batch 400 Loss 0.4253667891025543 Accuracy 0.39688435196876526\n",
      "Epoch 24 Batch 500 Loss 0.4327143430709839 Accuracy 0.3953690826892853\n",
      "Epoch 24  Loss 0.4331594705581665 Accuracy 0.3954806923866272\n",
      "Time take for 1 epoch:142.85889053344727 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-2     ************************\n",
      "\n",
      "Epoch 25 Batch 0 Loss 0.38136526942253113 Accuracy 0.40805289149284363\n",
      "Epoch 25 Batch 100 Loss 0.38133737444877625 Accuracy 0.413192480802536\n",
      "Epoch 25 Batch 200 Loss 0.39051488041877747 Accuracy 0.4087175726890564\n",
      "Epoch 25 Batch 300 Loss 0.39597198367118835 Accuracy 0.40434494614601135\n",
      "Epoch 25 Batch 400 Loss 0.40281063318252563 Accuracy 0.40113574266433716\n",
      "Epoch 25 Batch 500 Loss 0.4119848608970642 Accuracy 0.39867308735847473\n",
      "Epoch 25  Loss 0.41231516003608704 Accuracy 0.3986564576625824\n",
      "Time take for 1 epoch:142.98858499526978 secs\n",
      "\n",
      "Epoch 26 Batch 0 Loss 0.3245875835418701 Accuracy 0.3911830484867096\n",
      "Epoch 26 Batch 100 Loss 0.37152865529060364 Accuracy 0.4187982380390167\n",
      "Epoch 26 Batch 200 Loss 0.37590715289115906 Accuracy 0.4106449782848358\n",
      "Epoch 26 Batch 300 Loss 0.37980321049690247 Accuracy 0.4075186848640442\n",
      "Epoch 26 Batch 400 Loss 0.3871570825576782 Accuracy 0.4048588275909424\n",
      "Epoch 26 Batch 500 Loss 0.396223783493042 Accuracy 0.40199851989746094\n",
      "Epoch 26  Loss 0.39656296372413635 Accuracy 0.4021069407463074\n",
      "Time take for 1 epoch:142.94669818878174 secs\n",
      "\n",
      "Epoch 27 Batch 0 Loss 0.3298410475254059 Accuracy 0.4243749976158142\n",
      "Epoch 27 Batch 100 Loss 0.3528202176094055 Accuracy 0.4111378788948059\n",
      "Epoch 27 Batch 200 Loss 0.3555617928504944 Accuracy 0.4071243107318878\n",
      "Epoch 27 Batch 300 Loss 0.36267372965812683 Accuracy 0.40500563383102417\n",
      "Epoch 27 Batch 400 Loss 0.37133970856666565 Accuracy 0.4045761525630951\n",
      "Epoch 27 Batch 500 Loss 0.37918779253959656 Accuracy 0.4045071303844452\n",
      "Epoch 27  Loss 0.37921011447906494 Accuracy 0.4045158922672272\n",
      "Time take for 1 epoch:143.04092001914978 secs\n",
      "\n",
      "Epoch 28 Batch 0 Loss 0.31810301542282104 Accuracy 0.44437500834465027\n",
      "Epoch 28 Batch 100 Loss 0.33998942375183105 Accuracy 0.4163447916507721\n",
      "Epoch 28 Batch 200 Loss 0.3480130732059479 Accuracy 0.41535258293151855\n",
      "Epoch 28 Batch 300 Loss 0.3513295650482178 Accuracy 0.411258727312088\n",
      "Epoch 28 Batch 400 Loss 0.3587436378002167 Accuracy 0.40908533334732056\n",
      "Epoch 28 Batch 500 Loss 0.36529526114463806 Accuracy 0.4072165787220001\n",
      "Epoch 28  Loss 0.36537107825279236 Accuracy 0.4071636199951172\n",
      "Time take for 1 epoch:143.0739622116089 secs\n",
      "\n",
      "Epoch 29 Batch 0 Loss 0.29897159337997437 Accuracy 0.4073275923728943\n",
      "Epoch 29 Batch 100 Loss 0.32547295093536377 Accuracy 0.4190991520881653\n",
      "Epoch 29 Batch 200 Loss 0.3317614495754242 Accuracy 0.41692987084388733\n",
      "Epoch 29 Batch 300 Loss 0.3407941162586212 Accuracy 0.41592493653297424\n",
      "Epoch 29 Batch 400 Loss 0.3472760021686554 Accuracy 0.4139460325241089\n",
      "Epoch 29 Batch 500 Loss 0.35529401898384094 Accuracy 0.41164055466651917\n",
      "Epoch 29  Loss 0.35566645860671997 Accuracy 0.4116955101490021\n",
      "Time take for 1 epoch:142.85910153388977 secs\n",
      "\n",
      "Epoch 30 Batch 0 Loss 0.2611308991909027 Accuracy 0.3552083373069763\n",
      "Epoch 30 Batch 100 Loss 0.31259405612945557 Accuracy 0.4185478389263153\n",
      "Epoch 30 Batch 200 Loss 0.32124999165534973 Accuracy 0.4188821613788605\n",
      "Epoch 30 Batch 300 Loss 0.3272213935852051 Accuracy 0.41718339920043945\n",
      "Epoch 30 Batch 400 Loss 0.33559346199035645 Accuracy 0.4154357612133026\n",
      "Epoch 30 Batch 500 Loss 0.3411811292171478 Accuracy 0.41278448700904846\n",
      "Epoch 30  Loss 0.3413598835468292 Accuracy 0.41288939118385315\n",
      "Time take for 1 epoch:143.0463376045227 secs\n",
      "\n",
      "Epoch 31 Batch 0 Loss 0.3166350722312927 Accuracy 0.4579326808452606\n",
      "Epoch 31 Batch 100 Loss 0.30432558059692383 Accuracy 0.42307335138320923\n",
      "Epoch 31 Batch 200 Loss 0.30912917852401733 Accuracy 0.4195571839809418\n",
      "Epoch 31 Batch 300 Loss 0.3166109323501587 Accuracy 0.4211583137512207\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31 Batch 400 Loss 0.32514262199401855 Accuracy 0.4196661412715912\n",
      "Epoch 31 Batch 500 Loss 0.3314342498779297 Accuracy 0.41661351919174194\n",
      "Epoch 31  Loss 0.33167538046836853 Accuracy 0.4167594015598297\n",
      "Time take for 1 epoch:143.01716136932373 secs\n",
      "\n",
      "Epoch 32 Batch 0 Loss 0.25914469361305237 Accuracy 0.4288194477558136\n",
      "Epoch 32 Batch 100 Loss 0.29434478282928467 Accuracy 0.42731690406799316\n",
      "Epoch 32 Batch 200 Loss 0.2979860305786133 Accuracy 0.4228309094905853\n",
      "Epoch 32 Batch 300 Loss 0.30493080615997314 Accuracy 0.4206949472427368\n",
      "Epoch 32 Batch 400 Loss 0.31205272674560547 Accuracy 0.41990432143211365\n",
      "Epoch 32 Batch 500 Loss 0.31832361221313477 Accuracy 0.41824907064437866\n",
      "Epoch 32  Loss 0.31852078437805176 Accuracy 0.4182487428188324\n",
      "Time take for 1 epoch:143.06051206588745 secs\n",
      "\n",
      "Epoch 33 Batch 0 Loss 0.3101091682910919 Accuracy 0.4118303656578064\n",
      "Epoch 33 Batch 100 Loss 0.28221821784973145 Accuracy 0.42314720153808594\n",
      "Epoch 33 Batch 200 Loss 0.29090559482574463 Accuracy 0.42487508058547974\n",
      "Epoch 33 Batch 300 Loss 0.29589760303497314 Accuracy 0.424900084733963\n",
      "Epoch 33 Batch 400 Loss 0.30094245076179504 Accuracy 0.42126786708831787\n",
      "Epoch 33 Batch 500 Loss 0.30902794003486633 Accuracy 0.4201860725879669\n",
      "Epoch 33  Loss 0.3088558614253998 Accuracy 0.4199367165565491\n",
      "Time take for 1 epoch:143.04573440551758 secs\n",
      "\n",
      "Epoch 34 Batch 0 Loss 0.25607186555862427 Accuracy 0.42418980598449707\n",
      "Epoch 34 Batch 100 Loss 0.27907058596611023 Accuracy 0.43859511613845825\n",
      "Epoch 34 Batch 200 Loss 0.27929800748825073 Accuracy 0.4299837350845337\n",
      "Epoch 34 Batch 300 Loss 0.2858997583389282 Accuracy 0.42633116245269775\n",
      "Epoch 34 Batch 400 Loss 0.2920125126838684 Accuracy 0.4247407019138336\n",
      "Epoch 34 Batch 500 Loss 0.2989839017391205 Accuracy 0.42238402366638184\n",
      "Epoch 34  Loss 0.29926690459251404 Accuracy 0.42252999544143677\n",
      "Time take for 1 epoch:142.8450584411621 secs\n",
      "\n",
      "Epoch 35 Batch 0 Loss 0.22489967942237854 Accuracy 0.41724535822868347\n",
      "Epoch 35 Batch 100 Loss 0.2654593586921692 Accuracy 0.4303908348083496\n",
      "Epoch 35 Batch 200 Loss 0.27401575446128845 Accuracy 0.4307386577129364\n",
      "Epoch 35 Batch 300 Loss 0.2785973846912384 Accuracy 0.42787936329841614\n",
      "Epoch 35 Batch 400 Loss 0.2837485373020172 Accuracy 0.42504167556762695\n",
      "Epoch 35 Batch 500 Loss 0.2900044322013855 Accuracy 0.4243320822715759\n",
      "Epoch 35  Loss 0.290077269077301 Accuracy 0.42443254590034485\n",
      "Time take for 1 epoch:143.03061246871948 secs\n",
      "\n",
      "Epoch 36 Batch 0 Loss 0.24393901228904724 Accuracy 0.4340277910232544\n",
      "Epoch 36 Batch 100 Loss 0.2629961669445038 Accuracy 0.4329109191894531\n",
      "Epoch 36 Batch 200 Loss 0.26763004064559937 Accuracy 0.4318620264530182\n",
      "Epoch 36 Batch 300 Loss 0.27221912145614624 Accuracy 0.4303913414478302\n",
      "Epoch 36 Batch 400 Loss 0.2765747308731079 Accuracy 0.4274332523345947\n",
      "Epoch 36 Batch 500 Loss 0.28365471959114075 Accuracy 0.42709895968437195\n",
      "Epoch 36  Loss 0.2838488519191742 Accuracy 0.4271358549594879\n",
      "Time take for 1 epoch:142.81120920181274 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-3     ************************\n",
      "\n",
      "Epoch 37 Batch 0 Loss 0.24794769287109375 Accuracy 0.4806250035762787\n",
      "Epoch 37 Batch 100 Loss 0.2578180134296417 Accuracy 0.43745318055152893\n",
      "Epoch 37 Batch 200 Loss 0.2596495449542999 Accuracy 0.43412598967552185\n",
      "Epoch 37 Batch 300 Loss 0.2609516680240631 Accuracy 0.4282393157482147\n",
      "Epoch 37 Batch 400 Loss 0.2671445906162262 Accuracy 0.4283917248249054\n",
      "Epoch 37 Batch 500 Loss 0.27350953221321106 Accuracy 0.42697250843048096\n",
      "Epoch 37  Loss 0.27378687262535095 Accuracy 0.42713597416877747\n",
      "Time take for 1 epoch:142.7093322277069 secs\n",
      "\n",
      "Epoch 38 Batch 0 Loss 0.22085726261138916 Accuracy 0.4430803656578064\n",
      "Epoch 38 Batch 100 Loss 0.24514974653720856 Accuracy 0.43205153942108154\n",
      "Epoch 38 Batch 200 Loss 0.2489689290523529 Accuracy 0.4307705760002136\n",
      "Epoch 38 Batch 300 Loss 0.25536051392555237 Accuracy 0.431651771068573\n",
      "Epoch 38 Batch 400 Loss 0.26202964782714844 Accuracy 0.4316859841346741\n",
      "Epoch 38 Batch 500 Loss 0.26771217584609985 Accuracy 0.4294426739215851\n",
      "Epoch 38  Loss 0.2677031457424164 Accuracy 0.4291233420372009\n",
      "Time take for 1 epoch:142.6211404800415 secs\n",
      "\n",
      "Epoch 39 Batch 0 Loss 0.23882238566875458 Accuracy 0.47874999046325684\n",
      "Epoch 39 Batch 100 Loss 0.23739877343177795 Accuracy 0.4408498704433441\n",
      "Epoch 39 Batch 200 Loss 0.24191664159297943 Accuracy 0.43803104758262634\n",
      "Epoch 39 Batch 300 Loss 0.24639083445072174 Accuracy 0.43585842847824097\n",
      "Epoch 39 Batch 400 Loss 0.2529071867465973 Accuracy 0.4328122138977051\n",
      "Epoch 39 Batch 500 Loss 0.2598533630371094 Accuracy 0.4331119954586029\n",
      "Epoch 39  Loss 0.2599436938762665 Accuracy 0.4330720901489258\n",
      "Time take for 1 epoch:142.37569665908813 secs\n",
      "\n",
      "Epoch 40 Batch 0 Loss 0.2633759379386902 Accuracy 0.48750001192092896\n",
      "Epoch 40 Batch 100 Loss 0.23060446977615356 Accuracy 0.43108710646629333\n",
      "Epoch 40 Batch 200 Loss 0.24031497538089752 Accuracy 0.43799111247062683\n",
      "Epoch 40 Batch 300 Loss 0.24491895735263824 Accuracy 0.4369202256202698\n",
      "Epoch 40 Batch 400 Loss 0.24919918179512024 Accuracy 0.43351131677627563\n",
      "Epoch 40 Batch 500 Loss 0.2524663805961609 Accuracy 0.4309532940387726\n",
      "Epoch 40  Loss 0.25241485238075256 Accuracy 0.4308992922306061\n",
      "Time take for 1 epoch:142.90379667282104 secs\n",
      "\n",
      "Epoch 41 Batch 0 Loss 0.23120789229869843 Accuracy 0.5021306872367859\n",
      "Epoch 41 Batch 100 Loss 0.2224804013967514 Accuracy 0.43544650077819824\n",
      "Epoch 41 Batch 200 Loss 0.22981220483779907 Accuracy 0.439327597618103\n",
      "Epoch 41 Batch 300 Loss 0.23597167432308197 Accuracy 0.43753114342689514\n",
      "Epoch 41 Batch 400 Loss 0.2409183382987976 Accuracy 0.43569985032081604\n",
      "Epoch 41 Batch 500 Loss 0.24733150005340576 Accuracy 0.43412673473358154\n",
      "Epoch 41  Loss 0.24735581874847412 Accuracy 0.43418118357658386\n",
      "Time take for 1 epoch:142.65771770477295 secs\n",
      "\n",
      "Epoch 42 Batch 0 Loss 0.23441830277442932 Accuracy 0.5319293737411499\n",
      "Epoch 42 Batch 100 Loss 0.2231888771057129 Accuracy 0.44123098254203796\n",
      "Epoch 42 Batch 200 Loss 0.22576691210269928 Accuracy 0.4387035071849823\n",
      "Epoch 42 Batch 300 Loss 0.23088409006595612 Accuracy 0.43735581636428833\n",
      "Epoch 42 Batch 400 Loss 0.2353120595216751 Accuracy 0.43637675046920776\n",
      "Epoch 42 Batch 500 Loss 0.240025132894516 Accuracy 0.4343526363372803\n",
      "Epoch 42  Loss 0.24007269740104675 Accuracy 0.4343048334121704\n",
      "Time take for 1 epoch:142.66366910934448 secs\n",
      "\n",
      "Epoch 43 Batch 0 Loss 0.23801332712173462 Accuracy 0.4485677182674408\n",
      "Epoch 43 Batch 100 Loss 0.21598945558071136 Accuracy 0.4445911645889282\n",
      "Epoch 43 Batch 200 Loss 0.22161316871643066 Accuracy 0.4440091848373413\n",
      "Epoch 43 Batch 300 Loss 0.22602196037769318 Accuracy 0.4404888153076172\n",
      "Epoch 43 Batch 400 Loss 0.23081044852733612 Accuracy 0.4382283389568329\n",
      "Epoch 43 Batch 500 Loss 0.2348364293575287 Accuracy 0.43570607900619507\n",
      "Epoch 43  Loss 0.23489104211330414 Accuracy 0.435566782951355\n",
      "Time take for 1 epoch:142.64857506752014 secs\n",
      "\n",
      "Epoch 44 Batch 0 Loss 0.17670799791812897 Accuracy 0.4352678656578064\n",
      "Epoch 44 Batch 100 Loss 0.21061614155769348 Accuracy 0.4419221580028534\n",
      "Epoch 44 Batch 200 Loss 0.21785888075828552 Accuracy 0.4418865144252777\n",
      "Epoch 44 Batch 300 Loss 0.22092516720294952 Accuracy 0.44313758611679077\n",
      "Epoch 44 Batch 400 Loss 0.22404734790325165 Accuracy 0.4379396140575409\n",
      "Epoch 44 Batch 500 Loss 0.2295125126838684 Accuracy 0.43773946166038513\n",
      "Epoch 44  Loss 0.22972619533538818 Accuracy 0.4379015862941742\n",
      "Time take for 1 epoch:142.5203218460083 secs\n",
      "\n",
      "Epoch 45 Batch 0 Loss 0.1813267469406128 Accuracy 0.45703125\n",
      "Epoch 45 Batch 100 Loss 0.20734554529190063 Accuracy 0.4447978734970093\n",
      "Epoch 45 Batch 200 Loss 0.21460077166557312 Accuracy 0.44852352142333984\n",
      "Epoch 45 Batch 300 Loss 0.2185375988483429 Accuracy 0.4446824789047241\n",
      "Epoch 45 Batch 400 Loss 0.2225939929485321 Accuracy 0.44112488627433777\n",
      "Epoch 45 Batch 500 Loss 0.22512052953243256 Accuracy 0.4393834173679352\n",
      "Epoch 45  Loss 0.2252354621887207 Accuracy 0.4395911693572998\n",
      "Time take for 1 epoch:142.5249786376953 secs\n",
      "\n",
      "Epoch 46 Batch 0 Loss 0.20229165256023407 Accuracy 0.4324776828289032\n",
      "Epoch 46 Batch 100 Loss 0.20188269019126892 Accuracy 0.44525450468063354\n",
      "Epoch 46 Batch 200 Loss 0.20832635462284088 Accuracy 0.44470033049583435\n",
      "Epoch 46 Batch 300 Loss 0.21237978339195251 Accuracy 0.44556495547294617\n",
      "Epoch 46 Batch 400 Loss 0.21587620675563812 Accuracy 0.44238153100013733\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46 Batch 500 Loss 0.21997971832752228 Accuracy 0.4399130940437317\n",
      "Epoch 46  Loss 0.22002603113651276 Accuracy 0.4398233890533447\n",
      "Time take for 1 epoch:142.68507981300354 secs\n",
      "\n",
      "Epoch 47 Batch 0 Loss 0.19506266713142395 Accuracy 0.4046874940395355\n",
      "Epoch 47 Batch 100 Loss 0.1961705982685089 Accuracy 0.44268542528152466\n",
      "Epoch 47 Batch 200 Loss 0.20046138763427734 Accuracy 0.4432392120361328\n",
      "Epoch 47 Batch 300 Loss 0.20525570213794708 Accuracy 0.4430864751338959\n",
      "Epoch 47 Batch 400 Loss 0.20869049429893494 Accuracy 0.4409414827823639\n",
      "Epoch 47 Batch 500 Loss 0.21440713107585907 Accuracy 0.4410034716129303\n",
      "Epoch 47  Loss 0.21449682116508484 Accuracy 0.4408651888370514\n",
      "Time take for 1 epoch:142.74013423919678 secs\n",
      "\n",
      "Epoch 48 Batch 0 Loss 0.19022686779499054 Accuracy 0.4251077473163605\n",
      "Epoch 48 Batch 100 Loss 0.1950685977935791 Accuracy 0.4517563581466675\n",
      "Epoch 48 Batch 200 Loss 0.19844497740268707 Accuracy 0.4482550323009491\n",
      "Epoch 48 Batch 300 Loss 0.20103588700294495 Accuracy 0.44416341185569763\n",
      "Epoch 48 Batch 400 Loss 0.20543943345546722 Accuracy 0.443843275308609\n",
      "Epoch 48 Batch 500 Loss 0.21110756695270538 Accuracy 0.443770170211792\n",
      "Epoch 48  Loss 0.21106374263763428 Accuracy 0.44375139474868774\n",
      "Time take for 1 epoch:142.52289056777954 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-4     ************************\n",
      "\n",
      "Epoch 49 Batch 0 Loss 0.18261244893074036 Accuracy 0.4244791567325592\n",
      "Epoch 49 Batch 100 Loss 0.19169053435325623 Accuracy 0.4546610116958618\n",
      "Epoch 49 Batch 200 Loss 0.19465021789073944 Accuracy 0.44873538613319397\n",
      "Epoch 49 Batch 300 Loss 0.19985619187355042 Accuracy 0.4461008608341217\n",
      "Epoch 49 Batch 400 Loss 0.2029978334903717 Accuracy 0.4431139826774597\n",
      "Epoch 49 Batch 500 Loss 0.20702040195465088 Accuracy 0.4428471326828003\n",
      "Epoch 49  Loss 0.2071034461259842 Accuracy 0.4429987967014313\n",
      "Time take for 1 epoch:142.75779104232788 secs\n",
      "\n",
      "Epoch 50 Batch 0 Loss 0.19049614667892456 Accuracy 0.453125\n",
      "Epoch 50 Batch 100 Loss 0.1854921132326126 Accuracy 0.45281553268432617\n",
      "Epoch 50 Batch 200 Loss 0.1899348348379135 Accuracy 0.4504896104335785\n",
      "Epoch 50 Batch 300 Loss 0.19262413680553436 Accuracy 0.4476892948150635\n",
      "Epoch 50 Batch 400 Loss 0.1978106051683426 Accuracy 0.4479732811450958\n",
      "Epoch 50 Batch 500 Loss 0.20228995382785797 Accuracy 0.44631490111351013\n",
      "Epoch 50  Loss 0.2022644579410553 Accuracy 0.4462237060070038\n",
      "Time take for 1 epoch:142.46454691886902 secs\n",
      "\n",
      "Epoch 51 Batch 0 Loss 0.15791751444339752 Accuracy 0.44502314925193787\n",
      "Epoch 51 Batch 100 Loss 0.18347325921058655 Accuracy 0.445911705493927\n",
      "Epoch 51 Batch 200 Loss 0.18683882057666779 Accuracy 0.4436849057674408\n",
      "Epoch 51 Batch 300 Loss 0.19090555608272552 Accuracy 0.44550561904907227\n",
      "Epoch 51 Batch 400 Loss 0.19347935914993286 Accuracy 0.44394534826278687\n",
      "Epoch 51 Batch 500 Loss 0.19916851818561554 Accuracy 0.44480544328689575\n",
      "Epoch 51  Loss 0.19930511713027954 Accuracy 0.44491904973983765\n",
      "Time take for 1 epoch:142.60591793060303 secs\n",
      "\n",
      "Epoch 52 Batch 0 Loss 0.1620567888021469 Accuracy 0.4270833432674408\n",
      "Epoch 52 Batch 100 Loss 0.1828615367412567 Accuracy 0.4552271366119385\n",
      "Epoch 52 Batch 200 Loss 0.1850261390209198 Accuracy 0.45172369480133057\n",
      "Epoch 52 Batch 300 Loss 0.1867654025554657 Accuracy 0.44973403215408325\n",
      "Epoch 52 Batch 400 Loss 0.19007150828838348 Accuracy 0.44845956563949585\n",
      "Epoch 52 Batch 500 Loss 0.19383054971694946 Accuracy 0.4475807845592499\n",
      "Epoch 52  Loss 0.19399964809417725 Accuracy 0.44769537448883057\n",
      "Time take for 1 epoch:142.57004833221436 secs\n",
      "\n",
      "Epoch 53 Batch 0 Loss 0.17527782917022705 Accuracy 0.3917025923728943\n",
      "Epoch 53 Batch 100 Loss 0.17286556959152222 Accuracy 0.453177273273468\n",
      "Epoch 53 Batch 200 Loss 0.17628131806850433 Accuracy 0.44915804266929626\n",
      "Epoch 53 Batch 300 Loss 0.18026669323444366 Accuracy 0.448490709066391\n",
      "Epoch 53 Batch 400 Loss 0.18413221836090088 Accuracy 0.44887304306030273\n",
      "Epoch 53 Batch 500 Loss 0.18792873620986938 Accuracy 0.4477400779724121\n",
      "Epoch 53  Loss 0.1879497915506363 Accuracy 0.447611927986145\n",
      "Time take for 1 epoch:142.78711652755737 secs\n",
      "\n",
      "Epoch 54 Batch 0 Loss 0.19937939941883087 Accuracy 0.48828125\n",
      "Epoch 54 Batch 100 Loss 0.17171500623226166 Accuracy 0.4494505524635315\n",
      "Epoch 54 Batch 200 Loss 0.1737571507692337 Accuracy 0.4482736587524414\n",
      "Epoch 54 Batch 300 Loss 0.17953059077262878 Accuracy 0.4513934254646301\n",
      "Epoch 54 Batch 400 Loss 0.18160893023014069 Accuracy 0.4498211145401001\n",
      "Epoch 54 Batch 500 Loss 0.185346320271492 Accuracy 0.4488319158554077\n",
      "Epoch 54  Loss 0.18535713851451874 Accuracy 0.4488122761249542\n",
      "Time take for 1 epoch:142.7218325138092 secs\n",
      "\n",
      "Epoch 55 Batch 0 Loss 0.1552840918302536 Accuracy 0.44733795523643494\n",
      "Epoch 55 Batch 100 Loss 0.17108477652072906 Accuracy 0.46018844842910767\n",
      "Epoch 55 Batch 200 Loss 0.17164288461208344 Accuracy 0.45405882596969604\n",
      "Epoch 55 Batch 300 Loss 0.17574188113212585 Accuracy 0.45359405875205994\n",
      "Epoch 55 Batch 400 Loss 0.1790972650051117 Accuracy 0.45165911316871643\n",
      "Epoch 55 Batch 500 Loss 0.18297863006591797 Accuracy 0.4506937563419342\n",
      "Epoch 55  Loss 0.1829952895641327 Accuracy 0.4503837525844574\n",
      "Time take for 1 epoch:142.50209760665894 secs\n",
      "\n",
      "Epoch 56 Batch 0 Loss 0.14575974643230438 Accuracy 0.4177083373069763\n",
      "Epoch 56 Batch 100 Loss 0.16668400168418884 Accuracy 0.45513516664505005\n",
      "Epoch 56 Batch 200 Loss 0.1717361956834793 Accuracy 0.4575076103210449\n",
      "Epoch 56 Batch 300 Loss 0.17348572611808777 Accuracy 0.4546515941619873\n",
      "Epoch 56 Batch 400 Loss 0.1751713752746582 Accuracy 0.45095333456993103\n",
      "Epoch 56 Batch 500 Loss 0.1800786852836609 Accuracy 0.450575590133667\n",
      "Epoch 56  Loss 0.18016234040260315 Accuracy 0.45067620277404785\n",
      "Time take for 1 epoch:142.67448735237122 secs\n",
      "\n",
      "Epoch 57 Batch 0 Loss 0.15355966985225677 Accuracy 0.41582661867141724\n",
      "Epoch 57 Batch 100 Loss 0.1630972921848297 Accuracy 0.45723167061805725\n",
      "Epoch 57 Batch 200 Loss 0.16761690378189087 Accuracy 0.45700499415397644\n",
      "Epoch 57 Batch 300 Loss 0.17081688344478607 Accuracy 0.45581701397895813\n",
      "Epoch 57 Batch 400 Loss 0.17523202300071716 Accuracy 0.4557931125164032\n",
      "Epoch 57 Batch 500 Loss 0.17756880819797516 Accuracy 0.45259207487106323\n",
      "Epoch 57  Loss 0.17763018608093262 Accuracy 0.45262178778648376\n",
      "Time take for 1 epoch:142.4778687953949 secs\n",
      "\n",
      "Epoch 58 Batch 0 Loss 0.13976897299289703 Accuracy 0.45196759700775146\n",
      "Epoch 58 Batch 100 Loss 0.16118355095386505 Accuracy 0.45551279187202454\n",
      "Epoch 58 Batch 200 Loss 0.16299058496952057 Accuracy 0.45400577783584595\n",
      "Epoch 58 Batch 300 Loss 0.16736602783203125 Accuracy 0.45423370599746704\n",
      "Epoch 58 Batch 400 Loss 0.17113102972507477 Accuracy 0.45205724239349365\n",
      "Epoch 58 Batch 500 Loss 0.17460224032402039 Accuracy 0.4512229263782501\n",
      "Epoch 58  Loss 0.1746852844953537 Accuracy 0.4511064291000366\n",
      "Time take for 1 epoch:142.69250297546387 secs\n",
      "\n",
      "Epoch 59 Batch 0 Loss 0.14957991242408752 Accuracy 0.5115489363670349\n",
      "Epoch 59 Batch 100 Loss 0.15928976237773895 Accuracy 0.4633408784866333\n",
      "Epoch 59 Batch 200 Loss 0.16209682822227478 Accuracy 0.45995745062828064\n",
      "Epoch 59 Batch 300 Loss 0.16425152122974396 Accuracy 0.45915499329566956\n",
      "Epoch 59 Batch 400 Loss 0.1672607958316803 Accuracy 0.45710691809654236\n",
      "Epoch 59 Batch 500 Loss 0.1705447882413864 Accuracy 0.45527446269989014\n",
      "Epoch 59  Loss 0.17055851221084595 Accuracy 0.45525920391082764\n",
      "Time take for 1 epoch:142.37661480903625 secs\n",
      "\n",
      "Epoch 60 Batch 0 Loss 0.16946104168891907 Accuracy 0.49819710850715637\n",
      "Epoch 60 Batch 100 Loss 0.1594250351190567 Accuracy 0.46675240993499756\n",
      "Epoch 60 Batch 200 Loss 0.16123615205287933 Accuracy 0.4611141085624695\n",
      "Epoch 60 Batch 300 Loss 0.16397081315517426 Accuracy 0.46185463666915894\n",
      "Epoch 60 Batch 400 Loss 0.16612839698791504 Accuracy 0.45688626170158386\n",
      "Epoch 60 Batch 500 Loss 0.16843271255493164 Accuracy 0.45407047867774963\n",
      "Epoch 60  Loss 0.16854733228683472 Accuracy 0.4542897939682007\n",
      "Time take for 1 epoch:142.61226201057434 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-5     ************************\n",
      "\n",
      "Epoch 61 Batch 0 Loss 0.14451274275779724 Accuracy 0.4759615361690521\n",
      "Epoch 61 Batch 100 Loss 0.15213032066822052 Accuracy 0.46309319138526917\n",
      "Epoch 61 Batch 200 Loss 0.15550515055656433 Accuracy 0.45837897062301636\n",
      "Epoch 61 Batch 300 Loss 0.1574047952890396 Accuracy 0.45732447504997253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61 Batch 400 Loss 0.16079629957675934 Accuracy 0.45635896921157837\n",
      "Epoch 61 Batch 500 Loss 0.16467231512069702 Accuracy 0.4558199346065521\n",
      "Epoch 61  Loss 0.16479559242725372 Accuracy 0.45586952567100525\n",
      "Time take for 1 epoch:142.50131177902222 secs\n",
      "\n",
      "Epoch 62 Batch 0 Loss 0.17269407212734222 Accuracy 0.5143229365348816\n",
      "Epoch 62 Batch 100 Loss 0.15145191550254822 Accuracy 0.4618101119995117\n",
      "Epoch 62 Batch 200 Loss 0.15406197309494019 Accuracy 0.46008315682411194\n",
      "Epoch 62 Batch 300 Loss 0.1571657359600067 Accuracy 0.45798441767692566\n",
      "Epoch 62 Batch 400 Loss 0.16042490303516388 Accuracy 0.45794475078582764\n",
      "Epoch 62 Batch 500 Loss 0.16294489800930023 Accuracy 0.4563748836517334\n",
      "Epoch 62  Loss 0.16299088299274445 Accuracy 0.4564325511455536\n",
      "Time take for 1 epoch:142.51701140403748 secs\n",
      "\n",
      "Epoch 63 Batch 0 Loss 0.13263724744319916 Accuracy 0.4235491156578064\n",
      "Epoch 63 Batch 100 Loss 0.14956630766391754 Accuracy 0.46321481466293335\n",
      "Epoch 63 Batch 200 Loss 0.1505514532327652 Accuracy 0.4628719985485077\n",
      "Epoch 63 Batch 300 Loss 0.15414363145828247 Accuracy 0.4622425138950348\n",
      "Epoch 63 Batch 400 Loss 0.1577342003583908 Accuracy 0.45927250385284424\n",
      "Epoch 63 Batch 500 Loss 0.16035695374011993 Accuracy 0.45730870962142944\n",
      "Epoch 63  Loss 0.16035616397857666 Accuracy 0.4572477340698242\n",
      "Time take for 1 epoch:142.4992959499359 secs\n",
      "\n",
      "Epoch 64 Batch 0 Loss 0.12674593925476074 Accuracy 0.47295671701431274\n",
      "Epoch 64 Batch 100 Loss 0.14104951918125153 Accuracy 0.45904451608657837\n",
      "Epoch 64 Batch 200 Loss 0.14495187997817993 Accuracy 0.45942747592926025\n",
      "Epoch 64 Batch 300 Loss 0.14927025139331818 Accuracy 0.4579474925994873\n",
      "Epoch 64 Batch 400 Loss 0.15293271839618683 Accuracy 0.4560571312904358\n",
      "Epoch 64 Batch 500 Loss 0.1567508727312088 Accuracy 0.4560413360595703\n",
      "Epoch 64  Loss 0.15676775574684143 Accuracy 0.45586058497428894\n",
      "Time take for 1 epoch:142.8116593360901 secs\n",
      "\n",
      "Epoch 65 Batch 0 Loss 0.1192050352692604 Accuracy 0.4514508843421936\n",
      "Epoch 65 Batch 100 Loss 0.13873377442359924 Accuracy 0.4586373269557953\n",
      "Epoch 65 Batch 200 Loss 0.14425067603588104 Accuracy 0.45894184708595276\n",
      "Epoch 65 Batch 300 Loss 0.1482125222682953 Accuracy 0.4572674632072449\n",
      "Epoch 65 Batch 400 Loss 0.15161539614200592 Accuracy 0.45770204067230225\n",
      "Epoch 65 Batch 500 Loss 0.15476356446743011 Accuracy 0.45769500732421875\n",
      "Epoch 65  Loss 0.15490470826625824 Accuracy 0.45785588026046753\n",
      "Time take for 1 epoch:142.56487727165222 secs\n",
      "\n",
      "Epoch 66 Batch 0 Loss 0.16098040342330933 Accuracy 0.4918749928474426\n",
      "Epoch 66 Batch 100 Loss 0.14262890815734863 Accuracy 0.4621233344078064\n",
      "Epoch 66 Batch 200 Loss 0.1440812200307846 Accuracy 0.4622515141963959\n",
      "Epoch 66 Batch 300 Loss 0.14747697114944458 Accuracy 0.4619736075401306\n",
      "Epoch 66 Batch 400 Loss 0.1497923582792282 Accuracy 0.45801201462745667\n",
      "Epoch 66 Batch 500 Loss 0.15278273820877075 Accuracy 0.45785537362098694\n",
      "Epoch 66  Loss 0.15293090045452118 Accuracy 0.4580049216747284\n",
      "Time take for 1 epoch:142.53067827224731 secs\n",
      "\n",
      "Epoch 67 Batch 0 Loss 0.16140377521514893 Accuracy 0.5071022510528564\n",
      "Epoch 67 Batch 100 Loss 0.13934902846813202 Accuracy 0.4654775857925415\n",
      "Epoch 67 Batch 200 Loss 0.1405411809682846 Accuracy 0.462653785943985\n",
      "Epoch 67 Batch 300 Loss 0.1446298211812973 Accuracy 0.46432164311408997\n",
      "Epoch 67 Batch 400 Loss 0.14838089048862457 Accuracy 0.46281668543815613\n",
      "Epoch 67 Batch 500 Loss 0.15111155807971954 Accuracy 0.4605400860309601\n",
      "Epoch 67  Loss 0.15126805007457733 Accuracy 0.4605824649333954\n",
      "Time take for 1 epoch:142.32953524589539 secs\n",
      "\n",
      "Epoch 68 Batch 0 Loss 0.11847641319036484 Accuracy 0.4402901828289032\n",
      "Epoch 68 Batch 100 Loss 0.1393538862466812 Accuracy 0.4652397334575653\n",
      "Epoch 68 Batch 200 Loss 0.14058451354503632 Accuracy 0.4641237258911133\n",
      "Epoch 68 Batch 300 Loss 0.1431773453950882 Accuracy 0.4622759521007538\n",
      "Epoch 68 Batch 400 Loss 0.14548175036907196 Accuracy 0.4613659977912903\n",
      "Epoch 68 Batch 500 Loss 0.147745281457901 Accuracy 0.4601856470108032\n",
      "Epoch 68  Loss 0.14763973653316498 Accuracy 0.4600902795791626\n",
      "Time take for 1 epoch:142.4968225955963 secs\n",
      "\n",
      "Epoch 69 Batch 0 Loss 0.13972704112529755 Accuracy 0.4771634638309479\n",
      "Epoch 69 Batch 100 Loss 0.13423418998718262 Accuracy 0.4676549434661865\n",
      "Epoch 69 Batch 200 Loss 0.1342722326517105 Accuracy 0.4606873691082001\n",
      "Epoch 69 Batch 300 Loss 0.1380375176668167 Accuracy 0.4600811004638672\n",
      "Epoch 69 Batch 400 Loss 0.14190533757209778 Accuracy 0.46054866909980774\n",
      "Epoch 69 Batch 500 Loss 0.14474669098854065 Accuracy 0.4601987898349762\n",
      "Epoch 69  Loss 0.1448110193014145 Accuracy 0.4603074789047241\n",
      "Time take for 1 epoch:142.70365977287292 secs\n",
      "\n",
      "Epoch 70 Batch 0 Loss 0.09910091012716293 Accuracy 0.4291294515132904\n",
      "Epoch 70 Batch 100 Loss 0.13105161488056183 Accuracy 0.46317046880722046\n",
      "Epoch 70 Batch 200 Loss 0.13373199105262756 Accuracy 0.46198588609695435\n",
      "Epoch 70 Batch 300 Loss 0.13539843261241913 Accuracy 0.4587383270263672\n",
      "Epoch 70 Batch 400 Loss 0.13831588625907898 Accuracy 0.460198312997818\n",
      "Epoch 70 Batch 500 Loss 0.14136040210723877 Accuracy 0.45996537804603577\n",
      "Epoch 70  Loss 0.14141783118247986 Accuracy 0.4599124789237976\n",
      "Time take for 1 epoch:142.74178886413574 secs\n",
      "\n",
      "Epoch 71 Batch 0 Loss 0.14830616116523743 Accuracy 0.50390625\n",
      "Epoch 71 Batch 100 Loss 0.1290452480316162 Accuracy 0.4642135202884674\n",
      "Epoch 71 Batch 200 Loss 0.13333475589752197 Accuracy 0.46671196818351746\n",
      "Epoch 71 Batch 300 Loss 0.13539476692676544 Accuracy 0.4637816846370697\n",
      "Epoch 71 Batch 400 Loss 0.13836897909641266 Accuracy 0.4613770842552185\n",
      "Epoch 71 Batch 500 Loss 0.14205706119537354 Accuracy 0.46160760521888733\n",
      "Epoch 71  Loss 0.14222706854343414 Accuracy 0.46167463064193726\n",
      "Time take for 1 epoch:142.58754324913025 secs\n",
      "\n",
      "Epoch 72 Batch 0 Loss 0.1661272943019867 Accuracy 0.4925000071525574\n",
      "Epoch 72 Batch 100 Loss 0.13409703969955444 Accuracy 0.478058397769928\n",
      "Epoch 72 Batch 200 Loss 0.13481326401233673 Accuracy 0.46908628940582275\n",
      "Epoch 72 Batch 300 Loss 0.135864719748497 Accuracy 0.4670095145702362\n",
      "Epoch 72 Batch 400 Loss 0.13772107660770416 Accuracy 0.4639904797077179\n",
      "Epoch 72 Batch 500 Loss 0.1402290314435959 Accuracy 0.46236369013786316\n",
      "Epoch 72  Loss 0.14039826393127441 Accuracy 0.4623873829841614\n",
      "Time take for 1 epoch:142.52485370635986 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-6     ************************\n",
      "\n",
      "Epoch 73 Batch 0 Loss 0.14117489755153656 Accuracy 0.525390625\n",
      "Epoch 73 Batch 100 Loss 0.12634995579719543 Accuracy 0.46499302983283997\n",
      "Epoch 73 Batch 200 Loss 0.12985627353191376 Accuracy 0.46332308650016785\n",
      "Epoch 73 Batch 300 Loss 0.1330697387456894 Accuracy 0.46263375878334045\n",
      "Epoch 73 Batch 400 Loss 0.13628095388412476 Accuracy 0.46294844150543213\n",
      "Epoch 73 Batch 500 Loss 0.13865675032138824 Accuracy 0.4622665047645569\n",
      "Epoch 73  Loss 0.1386750191450119 Accuracy 0.46224865317344666\n",
      "Time take for 1 epoch:142.93610787391663 secs\n",
      "\n",
      "Epoch 74 Batch 0 Loss 0.10534146428108215 Accuracy 0.47187501192092896\n",
      "Epoch 74 Batch 100 Loss 0.12728771567344666 Accuracy 0.47123128175735474\n",
      "Epoch 74 Batch 200 Loss 0.12892065942287445 Accuracy 0.4680565297603607\n",
      "Epoch 74 Batch 300 Loss 0.13100819289684296 Accuracy 0.4670325517654419\n",
      "Epoch 74 Batch 400 Loss 0.13388194143772125 Accuracy 0.465509295463562\n",
      "Epoch 74 Batch 500 Loss 0.13607434928417206 Accuracy 0.4639037251472473\n",
      "Epoch 74  Loss 0.1361679583787918 Accuracy 0.46384885907173157\n",
      "Time take for 1 epoch:142.83902168273926 secs\n",
      "\n",
      "Epoch 75 Batch 0 Loss 0.13968771696090698 Accuracy 0.4412715435028076\n",
      "Epoch 75 Batch 100 Loss 0.12220793217420578 Accuracy 0.46923336386680603\n",
      "Epoch 75 Batch 200 Loss 0.12418904155492783 Accuracy 0.4627889394760132\n",
      "Epoch 75 Batch 300 Loss 0.12816016376018524 Accuracy 0.4645099937915802\n",
      "Epoch 75 Batch 400 Loss 0.1305089294910431 Accuracy 0.4638945162296295\n",
      "Epoch 75 Batch 500 Loss 0.13304182887077332 Accuracy 0.4633975028991699\n",
      "Epoch 75  Loss 0.1331886202096939 Accuracy 0.46340787410736084\n",
      "Time take for 1 epoch:143.0259153842926 secs\n",
      "\n",
      "Epoch 76 Batch 0 Loss 0.1333901435136795 Accuracy 0.4915364682674408\n",
      "Epoch 76 Batch 100 Loss 0.12266260385513306 Accuracy 0.4714415371417999\n",
      "Epoch 76 Batch 200 Loss 0.12457890063524246 Accuracy 0.46944665908813477\n",
      "Epoch 76 Batch 300 Loss 0.12738792598247528 Accuracy 0.46935638785362244\n",
      "Epoch 76 Batch 400 Loss 0.13003554940223694 Accuracy 0.466795414686203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76 Batch 500 Loss 0.1325121968984604 Accuracy 0.46458232402801514\n",
      "Epoch 76  Loss 0.13265712559223175 Accuracy 0.4647490382194519\n",
      "Time take for 1 epoch:142.85153222084045 secs\n",
      "\n",
      "Epoch 77 Batch 0 Loss 0.1159384623169899 Accuracy 0.5262500047683716\n",
      "Epoch 77 Batch 100 Loss 0.11892503499984741 Accuracy 0.46963751316070557\n",
      "Epoch 77 Batch 200 Loss 0.12036368995904922 Accuracy 0.468220591545105\n",
      "Epoch 77 Batch 300 Loss 0.12402272969484329 Accuracy 0.47022995352745056\n",
      "Epoch 77 Batch 400 Loss 0.12699346244335175 Accuracy 0.4684000015258789\n",
      "Epoch 77 Batch 500 Loss 0.1291029453277588 Accuracy 0.46598154306411743\n",
      "Epoch 77  Loss 0.12906542420387268 Accuracy 0.46579188108444214\n",
      "Time take for 1 epoch:142.85073947906494 secs\n",
      "\n",
      "Epoch 78 Batch 0 Loss 0.12077431380748749 Accuracy 0.5006250143051147\n",
      "Epoch 78 Batch 100 Loss 0.12048236280679703 Accuracy 0.474144846200943\n",
      "Epoch 78 Batch 200 Loss 0.12386985123157501 Accuracy 0.4709330201148987\n",
      "Epoch 78 Batch 300 Loss 0.12535567581653595 Accuracy 0.46963614225387573\n",
      "Epoch 78 Batch 400 Loss 0.1281474083662033 Accuracy 0.4667185842990875\n",
      "Epoch 78 Batch 500 Loss 0.12986324727535248 Accuracy 0.4659484326839447\n",
      "Epoch 78  Loss 0.12988562881946564 Accuracy 0.4660203456878662\n",
      "Time take for 1 epoch:142.88316440582275 secs\n",
      "\n",
      "Epoch 79 Batch 0 Loss 0.1391848623752594 Accuracy 0.5133101940155029\n",
      "Epoch 79 Batch 100 Loss 0.11394818127155304 Accuracy 0.46485093235969543\n",
      "Epoch 79 Batch 200 Loss 0.11831752210855484 Accuracy 0.46522074937820435\n",
      "Epoch 79 Batch 300 Loss 0.11902827769517899 Accuracy 0.46285519003868103\n",
      "Epoch 79 Batch 400 Loss 0.1228833869099617 Accuracy 0.46386146545410156\n",
      "Epoch 79 Batch 500 Loss 0.1256113499403 Accuracy 0.463895708322525\n",
      "Epoch 79  Loss 0.12565220892429352 Accuracy 0.46379488706588745\n",
      "Time take for 1 epoch:143.23547554016113 secs\n",
      "\n",
      "Epoch 80 Batch 0 Loss 0.10852517187595367 Accuracy 0.4513888955116272\n",
      "Epoch 80 Batch 100 Loss 0.11211206763982773 Accuracy 0.4611189067363739\n",
      "Epoch 80 Batch 200 Loss 0.116749607026577 Accuracy 0.4656412899494171\n",
      "Epoch 80 Batch 300 Loss 0.11930360645055771 Accuracy 0.46734318137168884\n",
      "Epoch 80 Batch 400 Loss 0.12267060577869415 Accuracy 0.4674118757247925\n",
      "Epoch 80 Batch 500 Loss 0.12489340454339981 Accuracy 0.4664400815963745\n",
      "Epoch 80  Loss 0.12493795156478882 Accuracy 0.46657246351242065\n",
      "Time take for 1 epoch:142.90346479415894 secs\n",
      "\n",
      "Epoch 81 Batch 0 Loss 0.09877101331949234 Accuracy 0.46238425374031067\n",
      "Epoch 81 Batch 100 Loss 0.1124894842505455 Accuracy 0.46922171115875244\n",
      "Epoch 81 Batch 200 Loss 0.11674582213163376 Accuracy 0.46909457445144653\n",
      "Epoch 81 Batch 300 Loss 0.11843515932559967 Accuracy 0.4647676646709442\n",
      "Epoch 81 Batch 400 Loss 0.1214827448129654 Accuracy 0.4677391052246094\n",
      "Epoch 81 Batch 500 Loss 0.12318159639835358 Accuracy 0.4667259156703949\n",
      "Epoch 81  Loss 0.12329964339733124 Accuracy 0.4666745960712433\n",
      "Time take for 1 epoch:142.8491358757019 secs\n",
      "\n",
      "Epoch 82 Batch 0 Loss 0.14338625967502594 Accuracy 0.5475260615348816\n",
      "Epoch 82 Batch 100 Loss 0.11508079618215561 Accuracy 0.46974262595176697\n",
      "Epoch 82 Batch 200 Loss 0.11686261743307114 Accuracy 0.4706318974494934\n",
      "Epoch 82 Batch 300 Loss 0.11978060752153397 Accuracy 0.4717091917991638\n",
      "Epoch 82 Batch 400 Loss 0.12122945487499237 Accuracy 0.4691010117530823\n",
      "Epoch 82 Batch 500 Loss 0.12280425429344177 Accuracy 0.466884970664978\n",
      "Epoch 82  Loss 0.12286697328090668 Accuracy 0.46693095564842224\n",
      "Time take for 1 epoch:142.94202661514282 secs\n",
      "\n",
      "Epoch 83 Batch 0 Loss 0.11726970970630646 Accuracy 0.4555288553237915\n",
      "Epoch 83 Batch 100 Loss 0.1109105572104454 Accuracy 0.4676177501678467\n",
      "Epoch 83 Batch 200 Loss 0.11313407123088837 Accuracy 0.47020667791366577\n",
      "Epoch 83 Batch 300 Loss 0.11514738202095032 Accuracy 0.4692939221858978\n",
      "Epoch 83 Batch 400 Loss 0.1179644912481308 Accuracy 0.469396710395813\n",
      "Epoch 83 Batch 500 Loss 0.11959220468997955 Accuracy 0.46650969982147217\n",
      "Epoch 83  Loss 0.1196088194847107 Accuracy 0.46629026532173157\n",
      "Time take for 1 epoch:143.05950665473938 secs\n",
      "\n",
      "Epoch 84 Batch 0 Loss 0.08806021511554718 Accuracy 0.4572916626930237\n",
      "Epoch 84 Batch 100 Loss 0.10908705741167068 Accuracy 0.4722932279109955\n",
      "Epoch 84 Batch 200 Loss 0.11090289056301117 Accuracy 0.4690758287906647\n",
      "Epoch 84 Batch 300 Loss 0.11466958373785019 Accuracy 0.47022080421447754\n",
      "Epoch 84 Batch 400 Loss 0.11704782396554947 Accuracy 0.4688860774040222\n",
      "Epoch 84 Batch 500 Loss 0.1198047623038292 Accuracy 0.4684733748435974\n",
      "Epoch 84  Loss 0.11981583386659622 Accuracy 0.46847298741340637\n",
      "Time take for 1 epoch:142.91341066360474 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-7     ************************\n",
      "\n",
      "Epoch 85 Batch 0 Loss 0.10730273276567459 Accuracy 0.44396552443504333\n",
      "Epoch 85 Batch 100 Loss 0.10982749611139297 Accuracy 0.46768122911453247\n",
      "Epoch 85 Batch 200 Loss 0.11066392064094543 Accuracy 0.4659824073314667\n",
      "Epoch 85 Batch 300 Loss 0.1129525676369667 Accuracy 0.46826446056365967\n",
      "Epoch 85 Batch 400 Loss 0.11465635150671005 Accuracy 0.4679498076438904\n",
      "Epoch 85 Batch 500 Loss 0.1167859211564064 Accuracy 0.4669541120529175\n",
      "Epoch 85  Loss 0.11687091737985611 Accuracy 0.466880202293396\n",
      "Time take for 1 epoch:143.08904910087585 secs\n",
      "\n",
      "Epoch 86 Batch 0 Loss 0.12331518530845642 Accuracy 0.5066105723381042\n",
      "Epoch 86 Batch 100 Loss 0.10994413495063782 Accuracy 0.4742695093154907\n",
      "Epoch 86 Batch 200 Loss 0.11072923988103867 Accuracy 0.4723701775074005\n",
      "Epoch 86 Batch 300 Loss 0.1122078225016594 Accuracy 0.46944668889045715\n",
      "Epoch 86 Batch 400 Loss 0.11427970975637436 Accuracy 0.4682121276855469\n",
      "Epoch 86 Batch 500 Loss 0.11625135689973831 Accuracy 0.467117041349411\n",
      "Epoch 86  Loss 0.11624996364116669 Accuracy 0.4670045077800751\n",
      "Time take for 1 epoch:143.09618091583252 secs\n",
      "\n",
      "Epoch 87 Batch 0 Loss 0.08272656798362732 Accuracy 0.4849759638309479\n",
      "Epoch 87 Batch 100 Loss 0.1064557284116745 Accuracy 0.47381773591041565\n",
      "Epoch 87 Batch 200 Loss 0.10806869715452194 Accuracy 0.47213223576545715\n",
      "Epoch 87 Batch 300 Loss 0.10949151962995529 Accuracy 0.4683607518672943\n",
      "Epoch 87 Batch 400 Loss 0.1121637225151062 Accuracy 0.46869298815727234\n",
      "Epoch 87 Batch 500 Loss 0.11463223397731781 Accuracy 0.4680410325527191\n",
      "Epoch 87  Loss 0.11465917527675629 Accuracy 0.4680889844894409\n",
      "Time take for 1 epoch:143.0068714618683 secs\n",
      "\n",
      "Epoch 88 Batch 0 Loss 0.10924903303384781 Accuracy 0.5089285969734192\n",
      "Epoch 88 Batch 100 Loss 0.10443896800279617 Accuracy 0.4706901013851166\n",
      "Epoch 88 Batch 200 Loss 0.1070811077952385 Accuracy 0.4694466292858124\n",
      "Epoch 88 Batch 300 Loss 0.10877344012260437 Accuracy 0.4693809151649475\n",
      "Epoch 88 Batch 400 Loss 0.11061713844537735 Accuracy 0.4689890742301941\n",
      "Epoch 88 Batch 500 Loss 0.11292386800050735 Accuracy 0.4694574177265167\n",
      "Epoch 88  Loss 0.11295394599437714 Accuracy 0.4693031311035156\n",
      "Time take for 1 epoch:142.93170952796936 secs\n",
      "\n",
      "Epoch 89 Batch 0 Loss 0.08930392563343048 Accuracy 0.359375\n",
      "Epoch 89 Batch 100 Loss 0.10706513375043869 Accuracy 0.47684967517852783\n",
      "Epoch 89 Batch 200 Loss 0.10826981067657471 Accuracy 0.4774210751056671\n",
      "Epoch 89 Batch 300 Loss 0.10992494970560074 Accuracy 0.4741070866584778\n",
      "Epoch 89 Batch 400 Loss 0.11180289089679718 Accuracy 0.47141432762145996\n",
      "Epoch 89 Batch 500 Loss 0.11364606022834778 Accuracy 0.4698941707611084\n",
      "Epoch 89  Loss 0.1136709526181221 Accuracy 0.46991682052612305\n",
      "Time take for 1 epoch:142.83164072036743 secs\n",
      "\n",
      "Epoch 90 Batch 0 Loss 0.10067741572856903 Accuracy 0.4897836446762085\n",
      "Epoch 90 Batch 100 Loss 0.10460192710161209 Accuracy 0.48251137137413025\n",
      "Epoch 90 Batch 200 Loss 0.10544832050800323 Accuracy 0.4768803119659424\n",
      "Epoch 90 Batch 300 Loss 0.10672815889120102 Accuracy 0.472049355506897\n",
      "Epoch 90 Batch 400 Loss 0.10875128954648972 Accuracy 0.4714232087135315\n",
      "Epoch 90 Batch 500 Loss 0.11065758019685745 Accuracy 0.47031378746032715\n",
      "Epoch 90  Loss 0.11069241911172867 Accuracy 0.47015953063964844\n",
      "Time take for 1 epoch:142.92223572731018 secs\n",
      "\n",
      "Epoch 91 Batch 0 Loss 0.08927159756422043 Accuracy 0.4084051847457886\n",
      "Epoch 91 Batch 100 Loss 0.10232141613960266 Accuracy 0.4721459746360779\n",
      "Epoch 91 Batch 200 Loss 0.10263793170452118 Accuracy 0.4709925055503845\n",
      "Epoch 91 Batch 300 Loss 0.10513366758823395 Accuracy 0.47295764088630676\n",
      "Epoch 91 Batch 400 Loss 0.10709495097398758 Accuracy 0.4719206392765045\n",
      "Epoch 91 Batch 500 Loss 0.10887809097766876 Accuracy 0.47043099999427795\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 91  Loss 0.10896775126457214 Accuracy 0.4705684781074524\n",
      "Time take for 1 epoch:142.87962007522583 secs\n",
      "\n",
      "Epoch 92 Batch 0 Loss 0.08404865860939026 Accuracy 0.42618533968925476\n",
      "Epoch 92 Batch 100 Loss 0.1013692170381546 Accuracy 0.4799376130104065\n",
      "Epoch 92 Batch 200 Loss 0.10330037027597427 Accuracy 0.4779853820800781\n",
      "Epoch 92 Batch 300 Loss 0.10501643270254135 Accuracy 0.47438448667526245\n",
      "Epoch 92 Batch 400 Loss 0.10764201730489731 Accuracy 0.47385990619659424\n",
      "Epoch 92 Batch 500 Loss 0.10906447470188141 Accuracy 0.47135359048843384\n",
      "Epoch 92  Loss 0.10904945433139801 Accuracy 0.4712105989456177\n",
      "Time take for 1 epoch:142.93009781837463 secs\n",
      "\n",
      "Epoch 93 Batch 0 Loss 0.09955502301454544 Accuracy 0.5377604365348816\n",
      "Epoch 93 Batch 100 Loss 0.09964119642972946 Accuracy 0.4755604565143585\n",
      "Epoch 93 Batch 200 Loss 0.10082155466079712 Accuracy 0.47481417655944824\n",
      "Epoch 93 Batch 300 Loss 0.10253703594207764 Accuracy 0.4713560938835144\n",
      "Epoch 93 Batch 400 Loss 0.1049313023686409 Accuracy 0.4718347489833832\n",
      "Epoch 93 Batch 500 Loss 0.10734323412179947 Accuracy 0.471316933631897\n",
      "Epoch 93  Loss 0.10736947506666183 Accuracy 0.47144025564193726\n",
      "Time take for 1 epoch:142.8662555217743 secs\n",
      "\n",
      "Epoch 94 Batch 0 Loss 0.10173217952251434 Accuracy 0.5093749761581421\n",
      "Epoch 94 Batch 100 Loss 0.09658381342887878 Accuracy 0.477518230676651\n",
      "Epoch 94 Batch 200 Loss 0.09839386492967606 Accuracy 0.4740437865257263\n",
      "Epoch 94 Batch 300 Loss 0.10135947912931442 Accuracy 0.4743496775627136\n",
      "Epoch 94 Batch 400 Loss 0.10363578051328659 Accuracy 0.4738231599330902\n",
      "Epoch 94 Batch 500 Loss 0.10557809472084045 Accuracy 0.4718506634235382\n",
      "Epoch 94  Loss 0.10568775236606598 Accuracy 0.47187891602516174\n",
      "Time take for 1 epoch:142.88639378547668 secs\n",
      "\n",
      "Epoch 95 Batch 0 Loss 0.09821510314941406 Accuracy 0.538722813129425\n",
      "Epoch 95 Batch 100 Loss 0.09523063898086548 Accuracy 0.4737327992916107\n",
      "Epoch 95 Batch 200 Loss 0.0987662822008133 Accuracy 0.4722554087638855\n",
      "Epoch 95 Batch 300 Loss 0.10055441409349442 Accuracy 0.47271108627319336\n",
      "Epoch 95 Batch 400 Loss 0.10205148160457611 Accuracy 0.4724281132221222\n",
      "Epoch 95 Batch 500 Loss 0.10415704548358917 Accuracy 0.47005850076675415\n",
      "Epoch 95  Loss 0.10419832170009613 Accuracy 0.4702690541744232\n",
      "Time take for 1 epoch:143.09729433059692 secs\n",
      "\n",
      "Epoch 96 Batch 0 Loss 0.09786481410264969 Accuracy 0.4442708194255829\n",
      "Epoch 96 Batch 100 Loss 0.09695009887218475 Accuracy 0.4788561463356018\n",
      "Epoch 96 Batch 200 Loss 0.09901883453130722 Accuracy 0.4731319844722748\n",
      "Epoch 96 Batch 300 Loss 0.09985269606113434 Accuracy 0.4719126522541046\n",
      "Epoch 96 Batch 400 Loss 0.1018497422337532 Accuracy 0.4709896445274353\n",
      "Epoch 96 Batch 500 Loss 0.10373198240995407 Accuracy 0.4710260033607483\n",
      "Epoch 96  Loss 0.10371198505163193 Accuracy 0.4710291028022766\n",
      "Time take for 1 epoch:143.12902736663818 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-8     ************************\n",
      "\n",
      "Epoch 97 Batch 0 Loss 0.09884866327047348 Accuracy 0.5088315010070801\n",
      "Epoch 97 Batch 100 Loss 0.09605354815721512 Accuracy 0.4753382205963135\n",
      "Epoch 97 Batch 200 Loss 0.09808961302042007 Accuracy 0.4758870601654053\n",
      "Epoch 97 Batch 300 Loss 0.10026033967733383 Accuracy 0.4764614701271057\n",
      "Epoch 97 Batch 400 Loss 0.10107333213090897 Accuracy 0.4734193682670593\n",
      "Epoch 97 Batch 500 Loss 0.10235564410686493 Accuracy 0.4730543792247772\n",
      "Epoch 97  Loss 0.10238387435674667 Accuracy 0.47305479645729065\n",
      "Time take for 1 epoch:142.78814363479614 secs\n",
      "\n",
      "Epoch 98 Batch 0 Loss 0.06559216231107712 Accuracy 0.4280133843421936\n",
      "Epoch 98 Batch 100 Loss 0.09648558497428894 Accuracy 0.47614794969558716\n",
      "Epoch 98 Batch 200 Loss 0.09772519767284393 Accuracy 0.4756660759449005\n",
      "Epoch 98 Batch 300 Loss 0.0990096926689148 Accuracy 0.475292831659317\n",
      "Epoch 98 Batch 400 Loss 0.0999705120921135 Accuracy 0.47323936223983765\n",
      "Epoch 98 Batch 500 Loss 0.10189128667116165 Accuracy 0.4725756049156189\n",
      "Epoch 98  Loss 0.10184422135353088 Accuracy 0.47245991230010986\n",
      "Time take for 1 epoch:143.00920224189758 secs\n",
      "\n",
      "Epoch 99 Batch 0 Loss 0.08808308839797974 Accuracy 0.39818549156188965\n",
      "Epoch 99 Batch 100 Loss 0.09396325796842575 Accuracy 0.47477033734321594\n",
      "Epoch 99 Batch 200 Loss 0.09592645615339279 Accuracy 0.4758538603782654\n",
      "Epoch 99 Batch 300 Loss 0.09716936200857162 Accuracy 0.4737570285797119\n",
      "Epoch 99 Batch 400 Loss 0.09975827485322952 Accuracy 0.4743827283382416\n",
      "Epoch 99 Batch 500 Loss 0.10170595347881317 Accuracy 0.4734687805175781\n",
      "Epoch 99  Loss 0.10176920145750046 Accuracy 0.4735959768295288\n",
      "Time take for 1 epoch:142.83599281311035 secs\n",
      "\n",
      "Epoch 100 Batch 0 Loss 0.10463757067918777 Accuracy 0.5358073115348816\n",
      "Epoch 100 Batch 100 Loss 0.09349797666072845 Accuracy 0.48107782006263733\n",
      "Epoch 100 Batch 200 Loss 0.09542481601238251 Accuracy 0.47787439823150635\n",
      "Epoch 100 Batch 300 Loss 0.09660036861896515 Accuracy 0.4774170517921448\n",
      "Epoch 100 Batch 400 Loss 0.09896297007799149 Accuracy 0.4771884083747864\n",
      "Epoch 100 Batch 500 Loss 0.10029606521129608 Accuracy 0.47520121932029724\n",
      "Epoch 100  Loss 0.10042637586593628 Accuracy 0.47512087225914\n",
      "Time take for 1 epoch:143.4001762866974 secs\n",
      "\n",
      "Epoch 101 Batch 0 Loss 0.09392891079187393 Accuracy 0.5224999785423279\n",
      "Epoch 101 Batch 100 Loss 0.09055831283330917 Accuracy 0.47723615169525146\n",
      "Epoch 101 Batch 200 Loss 0.0924195721745491 Accuracy 0.47357043623924255\n",
      "Epoch 101 Batch 300 Loss 0.09516119211912155 Accuracy 0.475112646818161\n",
      "Epoch 101 Batch 400 Loss 0.09728346765041351 Accuracy 0.474191278219223\n",
      "Epoch 101 Batch 500 Loss 0.09870312362909317 Accuracy 0.4740578532218933\n",
      "Epoch 101  Loss 0.0987248420715332 Accuracy 0.473964124917984\n",
      "Time take for 1 epoch:144.1328673362732 secs\n",
      "\n",
      "Epoch 102 Batch 0 Loss 0.07348743826150894 Accuracy 0.48006466031074524\n",
      "Epoch 102 Batch 100 Loss 0.08964082598686218 Accuracy 0.47522810101509094\n",
      "Epoch 102 Batch 200 Loss 0.09278199821710587 Accuracy 0.4744585156440735\n",
      "Epoch 102 Batch 300 Loss 0.09350506216287613 Accuracy 0.4753510653972626\n",
      "Epoch 102 Batch 400 Loss 0.09506937861442566 Accuracy 0.47390106320381165\n",
      "Epoch 102 Batch 500 Loss 0.0970640629529953 Accuracy 0.47381770610809326\n",
      "Epoch 102  Loss 0.09705857932567596 Accuracy 0.4738965630531311\n",
      "Time take for 1 epoch:144.41131949424744 secs\n",
      "\n",
      "Epoch 103 Batch 0 Loss 0.08246719092130661 Accuracy 0.4585336446762085\n",
      "Epoch 103 Batch 100 Loss 0.08891992270946503 Accuracy 0.4782184660434723\n",
      "Epoch 103 Batch 200 Loss 0.0903882086277008 Accuracy 0.47275909781455994\n",
      "Epoch 103 Batch 300 Loss 0.09188371896743774 Accuracy 0.4717557728290558\n",
      "Epoch 103 Batch 400 Loss 0.09410583972930908 Accuracy 0.47278282046318054\n",
      "Epoch 103 Batch 500 Loss 0.09642526507377625 Accuracy 0.47422149777412415\n",
      "Epoch 103  Loss 0.09645019471645355 Accuracy 0.47442907094955444\n",
      "Time take for 1 epoch:145.1817102432251 secs\n",
      "\n",
      "Epoch 104 Batch 0 Loss 0.09373018890619278 Accuracy 0.508152186870575\n",
      "Epoch 104 Batch 100 Loss 0.0883849710226059 Accuracy 0.47368207573890686\n",
      "Epoch 104 Batch 200 Loss 0.09013882279396057 Accuracy 0.4755282700061798\n",
      "Epoch 104 Batch 300 Loss 0.09182751178741455 Accuracy 0.4766218066215515\n",
      "Epoch 104 Batch 400 Loss 0.09325873106718063 Accuracy 0.47528934478759766\n",
      "Epoch 104 Batch 500 Loss 0.09496082365512848 Accuracy 0.47369760274887085\n",
      "Epoch 104  Loss 0.09497279673814774 Accuracy 0.4737038016319275\n",
      "Time take for 1 epoch:144.37103414535522 secs\n",
      "\n",
      "Epoch 105 Batch 0 Loss 0.07205890864133835 Accuracy 0.5443750023841858\n",
      "Epoch 105 Batch 100 Loss 0.08998294174671173 Accuracy 0.48213279247283936\n",
      "Epoch 105 Batch 200 Loss 0.09100550413131714 Accuracy 0.47807708382606506\n",
      "Epoch 105 Batch 300 Loss 0.09244029968976974 Accuracy 0.47640863060951233\n",
      "Epoch 105 Batch 400 Loss 0.09392203390598297 Accuracy 0.4769442081451416\n",
      "Epoch 105 Batch 500 Loss 0.09563406556844711 Accuracy 0.475601464509964\n",
      "Epoch 105  Loss 0.09569046646356583 Accuracy 0.4756722152233124\n",
      "Time take for 1 epoch:144.17051434516907 secs\n",
      "\n",
      "Epoch 106 Batch 0 Loss 0.08024632185697556 Accuracy 0.43803879618644714\n",
      "Epoch 106 Batch 100 Loss 0.08760303258895874 Accuracy 0.48413941264152527\n",
      "Epoch 106 Batch 200 Loss 0.08799666911363602 Accuracy 0.4763980805873871\n",
      "Epoch 106 Batch 300 Loss 0.09010511636734009 Accuracy 0.4756441116333008\n",
      "Epoch 106 Batch 400 Loss 0.09166455268859863 Accuracy 0.4741460084915161\n",
      "Epoch 106 Batch 500 Loss 0.09330666810274124 Accuracy 0.4743657112121582\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 106  Loss 0.09332184493541718 Accuracy 0.4742699861526489\n",
      "Time take for 1 epoch:144.29215812683105 secs\n",
      "\n",
      "Epoch 107 Batch 0 Loss 0.08740945160388947 Accuracy 0.4603794515132904\n",
      "Epoch 107 Batch 100 Loss 0.08694153279066086 Accuracy 0.4773235619068146\n",
      "Epoch 107 Batch 200 Loss 0.08868110924959183 Accuracy 0.47537967562675476\n",
      "Epoch 107 Batch 300 Loss 0.08965694159269333 Accuracy 0.4748607575893402\n",
      "Epoch 107 Batch 400 Loss 0.0912652462720871 Accuracy 0.475355863571167\n",
      "Epoch 107 Batch 500 Loss 0.09290175139904022 Accuracy 0.47432634234428406\n",
      "Epoch 107  Loss 0.09294001758098602 Accuracy 0.4743557274341583\n",
      "Time take for 1 epoch:144.38122510910034 secs\n",
      "\n",
      "Epoch 108 Batch 0 Loss 0.08239393681287766 Accuracy 0.46064814925193787\n",
      "Epoch 108 Batch 100 Loss 0.08545403182506561 Accuracy 0.4819352328777313\n",
      "Epoch 108 Batch 200 Loss 0.08563484996557236 Accuracy 0.4765799343585968\n",
      "Epoch 108 Batch 300 Loss 0.0882168784737587 Accuracy 0.47674503922462463\n",
      "Epoch 108 Batch 400 Loss 0.0901547446846962 Accuracy 0.4749082922935486\n",
      "Epoch 108 Batch 500 Loss 0.09190664440393448 Accuracy 0.47566962242126465\n",
      "Epoch 108  Loss 0.0918830931186676 Accuracy 0.4755402207374573\n",
      "Time take for 1 epoch:144.2295424938202 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-9     ************************\n",
      "\n",
      "Epoch 109 Batch 0 Loss 0.10507559031248093 Accuracy 0.47836539149284363\n",
      "Epoch 109 Batch 100 Loss 0.08683371543884277 Accuracy 0.4841974377632141\n",
      "Epoch 109 Batch 200 Loss 0.08716624975204468 Accuracy 0.4809403121471405\n",
      "Epoch 109 Batch 300 Loss 0.08815083652734756 Accuracy 0.4806371033191681\n",
      "Epoch 109 Batch 400 Loss 0.08956437557935715 Accuracy 0.4778591990470886\n",
      "Epoch 109 Batch 500 Loss 0.09140085428953171 Accuracy 0.477439284324646\n",
      "Epoch 109  Loss 0.09144026786088943 Accuracy 0.47735050320625305\n",
      "Time take for 1 epoch:144.16425561904907 secs\n",
      "\n",
      "Epoch 110 Batch 0 Loss 0.09492158889770508 Accuracy 0.45432692766189575\n",
      "Epoch 110 Batch 100 Loss 0.08501701802015305 Accuracy 0.4852999746799469\n",
      "Epoch 110 Batch 200 Loss 0.08540451526641846 Accuracy 0.48175424337387085\n",
      "Epoch 110 Batch 300 Loss 0.08704837411642075 Accuracy 0.4781171977519989\n",
      "Epoch 110 Batch 400 Loss 0.0885085016489029 Accuracy 0.47670331597328186\n",
      "Epoch 110 Batch 500 Loss 0.09057121723890305 Accuracy 0.4764372706413269\n",
      "Epoch 110  Loss 0.09061110019683838 Accuracy 0.47630947828292847\n",
      "Time take for 1 epoch:144.2063455581665 secs\n",
      "\n",
      "Epoch 111 Batch 0 Loss 0.07176001369953156 Accuracy 0.4319196343421936\n",
      "Epoch 111 Batch 100 Loss 0.08205139636993408 Accuracy 0.4773228168487549\n",
      "Epoch 111 Batch 200 Loss 0.08299902826547623 Accuracy 0.4769488573074341\n",
      "Epoch 111 Batch 300 Loss 0.08491005003452301 Accuracy 0.4775788187980652\n",
      "Epoch 111 Batch 400 Loss 0.08662491291761398 Accuracy 0.47624558210372925\n",
      "Epoch 111 Batch 500 Loss 0.08829187601804733 Accuracy 0.474492609500885\n",
      "Epoch 111  Loss 0.0883377343416214 Accuracy 0.47461092472076416\n",
      "Time take for 1 epoch:144.39647936820984 secs\n",
      "\n",
      "Epoch 112 Batch 0 Loss 0.07849451899528503 Accuracy 0.48206019401550293\n",
      "Epoch 112 Batch 100 Loss 0.07992823421955109 Accuracy 0.4813092350959778\n",
      "Epoch 112 Batch 200 Loss 0.0825769230723381 Accuracy 0.48227760195732117\n",
      "Epoch 112 Batch 300 Loss 0.0847213938832283 Accuracy 0.4792875647544861\n",
      "Epoch 112 Batch 400 Loss 0.08723313361406326 Accuracy 0.4802582859992981\n",
      "Epoch 112 Batch 500 Loss 0.08884888142347336 Accuracy 0.4792705476284027\n",
      "Epoch 112  Loss 0.08889608085155487 Accuracy 0.47917571663856506\n",
      "Time take for 1 epoch:143.75340580940247 secs\n",
      "\n",
      "Epoch 113 Batch 0 Loss 0.07116571068763733 Accuracy 0.47511574625968933\n",
      "Epoch 113 Batch 100 Loss 0.08155720680952072 Accuracy 0.4845738112926483\n",
      "Epoch 113 Batch 200 Loss 0.08168736100196838 Accuracy 0.4808933436870575\n",
      "Epoch 113 Batch 300 Loss 0.08339061588048935 Accuracy 0.4782949388027191\n",
      "Epoch 113 Batch 400 Loss 0.08545087277889252 Accuracy 0.4785740375518799\n",
      "Epoch 113 Batch 500 Loss 0.08722946047782898 Accuracy 0.4779511094093323\n",
      "Epoch 113  Loss 0.0872575044631958 Accuracy 0.4781237244606018\n",
      "Time take for 1 epoch:142.78544783592224 secs\n",
      "\n",
      "Epoch 114 Batch 0 Loss 0.07859627157449722 Accuracy 0.4934895932674408\n",
      "Epoch 114 Batch 100 Loss 0.07995451241731644 Accuracy 0.4703878164291382\n",
      "Epoch 114 Batch 200 Loss 0.08240688592195511 Accuracy 0.4761936664581299\n",
      "Epoch 114 Batch 300 Loss 0.08359559625387192 Accuracy 0.47558921575546265\n",
      "Epoch 114 Batch 400 Loss 0.08602435141801834 Accuracy 0.4772283732891083\n",
      "Epoch 114 Batch 500 Loss 0.08718708157539368 Accuracy 0.4765373766422272\n",
      "Epoch 114  Loss 0.08728582412004471 Accuracy 0.47662925720214844\n",
      "Time take for 1 epoch:142.91222667694092 secs\n",
      "\n",
      "Epoch 115 Batch 0 Loss 0.07848502695560455 Accuracy 0.4654017984867096\n",
      "Epoch 115 Batch 100 Loss 0.07950343936681747 Accuracy 0.47626256942749023\n",
      "Epoch 115 Batch 200 Loss 0.08076883852481842 Accuracy 0.4777868390083313\n",
      "Epoch 115 Batch 300 Loss 0.0822906494140625 Accuracy 0.4761952757835388\n",
      "Epoch 115 Batch 400 Loss 0.08417635411024094 Accuracy 0.4762676954269409\n",
      "Epoch 115 Batch 500 Loss 0.08540476858615875 Accuracy 0.4772425591945648\n",
      "Epoch 115  Loss 0.08543524146080017 Accuracy 0.4773694574832916\n",
      "Time take for 1 epoch:143.03497052192688 secs\n",
      "\n",
      "Epoch 116 Batch 0 Loss 0.09578169137239456 Accuracy 0.4585336446762085\n",
      "Epoch 116 Batch 100 Loss 0.08073658496141434 Accuracy 0.4838467240333557\n",
      "Epoch 116 Batch 200 Loss 0.08107393234968185 Accuracy 0.47701025009155273\n",
      "Epoch 116 Batch 300 Loss 0.08156971633434296 Accuracy 0.4753294587135315\n",
      "Epoch 116 Batch 400 Loss 0.08336490392684937 Accuracy 0.47693201899528503\n",
      "Epoch 116 Batch 500 Loss 0.0847897082567215 Accuracy 0.4778438210487366\n",
      "Epoch 116  Loss 0.08488068729639053 Accuracy 0.47778528928756714\n",
      "Time take for 1 epoch:142.92060232162476 secs\n",
      "\n",
      "Epoch 117 Batch 0 Loss 0.08523743599653244 Accuracy 0.4208669364452362\n",
      "Epoch 117 Batch 100 Loss 0.07850836217403412 Accuracy 0.4849908649921417\n",
      "Epoch 117 Batch 200 Loss 0.07971607893705368 Accuracy 0.48173466324806213\n",
      "Epoch 117 Batch 300 Loss 0.08192069083452225 Accuracy 0.4819183051586151\n",
      "Epoch 117 Batch 400 Loss 0.08328632265329361 Accuracy 0.4796128273010254\n",
      "Epoch 117 Batch 500 Loss 0.08488716930150986 Accuracy 0.47840890288352966\n",
      "Epoch 117  Loss 0.08498428761959076 Accuracy 0.4785684049129486\n",
      "Time take for 1 epoch:142.87083411216736 secs\n",
      "\n",
      "Epoch 118 Batch 0 Loss 0.07737408578395844 Accuracy 0.5\n",
      "Epoch 118 Batch 100 Loss 0.0793822854757309 Accuracy 0.48467424511909485\n",
      "Epoch 118 Batch 200 Loss 0.08015041798353195 Accuracy 0.47723597288131714\n",
      "Epoch 118 Batch 300 Loss 0.08068320900201797 Accuracy 0.47902241349220276\n",
      "Epoch 118 Batch 400 Loss 0.08297654241323471 Accuracy 0.48045113682746887\n",
      "Epoch 118 Batch 500 Loss 0.08437381684780121 Accuracy 0.47919636964797974\n",
      "Epoch 118  Loss 0.08439934253692627 Accuracy 0.4791187644004822\n",
      "Time take for 1 epoch:142.8178322315216 secs\n",
      "\n",
      "Epoch 119 Batch 0 Loss 0.08480704575777054 Accuracy 0.5305989384651184\n",
      "Epoch 119 Batch 100 Loss 0.07705164700746536 Accuracy 0.4890022277832031\n",
      "Epoch 119 Batch 200 Loss 0.07783922553062439 Accuracy 0.4826916754245758\n",
      "Epoch 119 Batch 300 Loss 0.07831248641014099 Accuracy 0.4804351031780243\n",
      "Epoch 119 Batch 400 Loss 0.08091848343610764 Accuracy 0.4802299737930298\n",
      "Epoch 119 Batch 500 Loss 0.08219843357801437 Accuracy 0.4785030484199524\n",
      "Epoch 119  Loss 0.08221787959337234 Accuracy 0.4785816967487335\n",
      "Time take for 1 epoch:143.03249788284302 secs\n",
      "\n",
      "Epoch 120 Batch 0 Loss 0.07719072699546814 Accuracy 0.45370370149612427\n",
      "Epoch 120 Batch 100 Loss 0.07609523832798004 Accuracy 0.47690367698669434\n",
      "Epoch 120 Batch 200 Loss 0.07748178392648697 Accuracy 0.4786415696144104\n",
      "Epoch 120 Batch 300 Loss 0.07924682646989822 Accuracy 0.47900617122650146\n",
      "Epoch 120 Batch 400 Loss 0.08083906769752502 Accuracy 0.47954660654067993\n",
      "Epoch 120 Batch 500 Loss 0.08227431029081345 Accuracy 0.4788963496685028\n",
      "Epoch 120  Loss 0.0822380855679512 Accuracy 0.4789828062057495\n",
      "Time take for 1 epoch:142.94644904136658 secs\n",
      "\n",
      "**********************    model saved to ./checkpoint/trf-10     ************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# **************************************  模型训练   *********************************************\n",
    "# 1、初始化模型\n",
    "input_vocab_size = pt_tokenizer.vocab_size + 2\n",
    "target_vocab_size = en_tokenizer.vocab_size + 2\n",
    "d_model = 128\n",
    "max_len = 40\n",
    "heads_num = 8\n",
    "dff = 512\n",
    "layers_num = 4\n",
    "transformer = Transformer(input_vocab_size, target_vocab_size, d_model, max_len, heads_num, dff, layers_num)\n",
    "\n",
    "# 2、自定义损失函数\n",
    "def loss_func(real,pred):\n",
    "    '''\n",
    "    :param real: shape:(batch_size, target_seq_len)\n",
    "    :param pred: shape: (batch_size, target_seq_len, target_vocab_size)\n",
    "    :return:\n",
    "    '''\n",
    "    mask = 1-tf.cast(tf.math.equal(real, 0),tf.float32) # shape: (batch_size, target_seq_len)\n",
    "    loss_function = tf.losses.SparseCategoricalCrossentropy(from_logits=False, reduction='none')\n",
    "    loss = loss_function(real,pred) # shape: (batch_size, target_seq_len)\n",
    "    return tf.reduce_mean(tf.multiply(loss, mask))\n",
    "\n",
    "# 3、自定义学习率\n",
    "# lrate = (d_model **-0.5) * min(step_num **-0.5,step_num*warm_up_steps**-1.5) 先增后减的学习率\n",
    "class CustomizedSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warm_up_steps=4000):\n",
    "        super(CustomizedSchedule, self).__init__()\n",
    "        self.d_model = tf.cast(d_model, tf.float32)\n",
    "        self.warm_up_steps = warm_up_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warm_up_steps ** (-1.5))\n",
    "        arg3 = tf.math.rsqrt(self.d_model)\n",
    "        return arg3 * tf.math.minimum(arg1, arg2)\n",
    "# # 学习率曲线绘图\n",
    "# temp_learning_rate_schedule = CustomizedSchedule(d_model)\n",
    "# # 0-40000步\n",
    "# plt.plot(temp_learning_rate_schedule(tf.range(40000,dtype=tf.float32)))\n",
    "# plt.ylabel(\"Learning Rate\")\n",
    "# plt.xlabel(\"train_step\")\n",
    "\n",
    "\n",
    "# 4、optimizer优化器\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=CustomizedSchedule(d_model))\n",
    "\n",
    "# 5、定义评估指标\n",
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
    "\n",
    "# 6、定义好 checkpoint\n",
    "checkpoint = tf.train.Checkpoint(model=transformer,\n",
    "                                optimizer=optimizer)\n",
    "\n",
    "# 7、定义train_step\n",
    "train_step_signature = [\n",
    "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
    "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
    "]\n",
    "@tf.function(input_signature=train_step_signature)\n",
    "def train_step(inp, target):\n",
    "    tar_inp = target[:,:-1]\n",
    "    tar_real = target[:,1:]\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = transformer(inp, tar_inp, training=True)\n",
    "        loss = loss_func(tar_real, predictions)\n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "    train_loss(loss)\n",
    "    train_accuracy(tar_real, predictions)\n",
    "\n",
    "\n",
    "# 8、循环执行训练流程\n",
    "import time\n",
    "epoches = 120\n",
    "for epoch in range(epoches):\n",
    "    start = time.time()\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "    for (batch, (inp, target)) in enumerate(train_datasets):\n",
    "        train_step(inp,target)\n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {} Accuracy {}'.format(epoch+1,batch,train_loss.result(),train_accuracy.result()))\n",
    "            \n",
    "    print('Epoch {}  Loss {} Accuracy {}'.format(epoch + 1, train_loss.result(), train_accuracy.result()))\n",
    "    print('Time take for 1 epoch:{} secs\\n'.format(time.time()-start))\n",
    "    \n",
    "    if (epoch+1) % 12 == 0:\n",
    "        path = checkpoint.save(file_prefix='./checkpoint/trf')\n",
    "        print(\"**********************    model saved to %s     ************************\" % path)\n",
    "        print()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:55:28.895453Z",
     "start_time": "2021-09-17T02:55:27.799459Z"
    }
   },
   "outputs": [],
   "source": [
    "transformer.save_weights('./saved model/transformer.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5、预测与使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:56:30.863106Z",
     "start_time": "2021-09-17T02:56:30.850094Z"
    }
   },
   "outputs": [],
   "source": [
    "def evalute(inp_sentence, model):\n",
    "    input_id_sentence = [pt_tokenizer.vocab_size]+pt_tokenizer.encode(inp_sentence)+ [pt_tokenizer.vocab_size+1]\n",
    "    encoder_input =tf.expand_dims(input_id_sentence,0) # (1,input_sentence_length)\n",
    "    decoder_input = tf.expand_dims([en_tokenizer.vocab_size],0) # (1,1)\n",
    "    for i in range(max_len):\n",
    "#         encoder_padding_mask ,decoder_mask,encoder_decoder_padding_mask = create_mask(encoder_input,decoder_input)\n",
    "        #(batch_size,output_target_len,target_vocab_size)\n",
    "        predictions = model(encoder_input, decoder_input, training=False)\n",
    "        predictions = predictions[:,-1,:] # 单步\n",
    "        predictions_id = tf.cast(tf.argmax(predictions,axis=-1),tf.int32) #预测概率最大的值\n",
    "        if tf.equal(predictions_id,en_tokenizer.vocab_size+1):\n",
    "            return tf.squeeze(decoder_input,axis=0)\n",
    "        \n",
    "        decoder_input = tf.concat([decoder_input,[predictions_id]],\n",
    "                                  axis=-1)\n",
    "    return tf.squeeze(decoder_input,axis=0)\n",
    "\n",
    "def translate(input_sentence, model):\n",
    "    result = evalute(input_sentence, model)\n",
    "    predicted_sentence = en_tokenizer.decode([i for i in result if i < en_tokenizer.vocab_size]) # 防止无词出错\n",
    "    print(\"Input: {}\".format(input_sentence))\n",
    "    print(\"Predicted translation: {}\".format(predicted_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T03:11:57.541314Z",
     "start_time": "2021-09-17T03:11:57.523299Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"transformer_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "encoder_model_2 (EncoderMode multiple                  1828480   \n",
      "_________________________________________________________________\n",
      "decoder_model_2 (DecoderMode multiple                  2093696   \n",
      "_________________________________________________________________\n",
      "dense_194 (Dense)            multiple                  1043481   \n",
      "=================================================================\n",
      "Total params: 4,965,657\n",
      "Trainable params: 4,965,657\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "transformer.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T06:52:23.965457Z",
     "start_time": "2021-09-16T06:52:23.837339Z"
    }
   },
   "outputs": [],
   "source": [
    "transformer.load_weights('./saved model/transformer.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:56:40.962290Z",
     "start_time": "2021-09-17T02:56:40.530897Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: obrigado .\n",
      "Predicted translation: thank you .\n"
     ]
    }
   ],
   "source": [
    "# 正式使用\n",
    "input_sentence = 'obrigado .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:56:46.882683Z",
     "start_time": "2021-09-17T02:56:46.217077Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: a cirurgia foi um sucesso .\n",
      "Predicted translation: the surgery was successful .\n"
     ]
    }
   ],
   "source": [
    "# 正式使用\n",
    "input_sentence = 'a cirurgia foi um sucesso .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:58:23.006121Z",
     "start_time": "2021-09-17T02:58:22.002197Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: este é o primeiro livro que já tenho .\n",
      "Predicted translation: that 's the first book i have .\n"
     ]
    }
   ],
   "source": [
    "# 正式使用\n",
    "input_sentence = 'Este é o primeiro livro que já tenho .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:58:30.261708Z",
     "start_time": "2021-09-17T02:58:29.628131Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: estou muito feliz .\n",
      "Predicted translation: i am so happy .\n"
     ]
    }
   ],
   "source": [
    " # 正式使用\n",
    "input_sentence = 'Estou Muito feliz .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:58:48.731025Z",
     "start_time": "2021-09-17T02:58:48.121470Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input:  você é estúpido .\n",
      "Predicted translation: you 're stupid .\n"
     ]
    }
   ],
   "source": [
    " # 正式使用\n",
    "input_sentence = ' Você é estúpido .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:59:19.533038Z",
     "start_time": "2021-09-17T02:59:18.605194Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: você é tudo para mim .\n",
      "Predicted translation: you 're all meant to me .\n"
     ]
    }
   ],
   "source": [
    "# 正式使用\n",
    "input_sentence = 'Você é tudo para mim .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T03:06:55.047707Z",
     "start_time": "2021-09-17T03:06:54.412119Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: eu sempre te amarei .\n",
      "Predicted translation: i always love you .\n"
     ]
    }
   ],
   "source": [
    "# 正式使用\n",
    "input_sentence = 'Eu sempre te amarei .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T02:59:47.722005Z",
     "start_time": "2021-09-17T02:59:47.116444Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: quem é o melhor ?\n",
      "Predicted translation: who is the best ?\n"
     ]
    }
   ],
   "source": [
    "  \n",
    "# 正式使用\n",
    "input_sentence = 'Quem é o melhor ?'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-17T03:03:30.706490Z",
     "start_time": "2021-09-17T03:03:29.947799Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: eu acredito que posso voar .\n",
      "Predicted translation: i believe i can fly .\n"
     ]
    }
   ],
   "source": [
    " \n",
    "# 正式使用\n",
    "input_sentence = 'Eu acredito que posso voar .'\n",
    "translate(input_sentence.lower(),transformer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6、断点续训"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-16T18:05:54.501073Z",
     "start_time": "2021-09-16T18:04:48.345313Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 3.401538848876953 Accuracy 0.09730113297700882\n",
      "Epoch 1 Batch 100 Loss 3.047414541244507 Accuracy 0.0977599024772644\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-bffb5069c600>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mtrain_accuracy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_states\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0minp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_datasets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m         \u001b[0mtrain_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minp\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbatch\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m100\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Epoch {} Batch {} Loss {} Accuracy {}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtrain_accuracy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    566\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    567\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 568\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    569\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    570\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    597\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    598\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 599\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    600\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    601\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2361\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2362\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2363\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2365\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs)\u001b[0m\n\u001b[0;32m   1609\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[0;32m   1610\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[1;32m-> 1611\u001b[1;33m         self.captured_inputs)\n\u001b[0m\u001b[0;32m   1612\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1613\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1690\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1691\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1692\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1693\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1694\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mD:\\Anaconda\\envs\\TF2.1\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[0;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 61\u001b[1;33m                                                num_outputs)\n\u001b[0m\u001b[0;32m     62\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 模型参数配置\n",
    "input_vocab_size = pt_tokenizer.vocab_size + 2\n",
    "target_vocab_size = en_tokenizer.vocab_size + 2\n",
    "d_model = 128\n",
    "max_len = 40\n",
    "heads_num = 8\n",
    "dff = 512\n",
    "layers_num = 4\n",
    "\n",
    "def continue_train():\n",
    "    '''\n",
    "    如果要在本文件中调用，\n",
    "    '''\n",
    "    # 1、创建模型\n",
    "    transformer = Transformer(input_vocab_size, target_vocab_size, d_model, max_len, heads_num, dff, layers_num)\n",
    "    # 2、创建optimizer优化器（学习率用上面定义过的）\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=CustomizedSchedule(d_model))\n",
    "    # 3、创建checkpoint对象，加载模型与其优化器\n",
    "    checkpoint = tf.train.Checkpoint(model=transformer,\n",
    "                                    optimizer=optimizer)\n",
    "    # 4、将 checkpoint文件信息，恢复到model与优化器上\n",
    "    checkpoint.restore(tf.train.latest_checkpoint('./checkpoint/'))\n",
    "    \n",
    "    # 5、此时model已经恢复，可以续训\n",
    "    epoches = 20\n",
    "    for epoch in range(epoches):\n",
    "        start = time.time()\n",
    "        train_loss.reset_states()\n",
    "        train_accuracy.reset_states()\n",
    "        for (batch, (inp, target)) in enumerate(train_datasets):\n",
    "            train_step(inp,target)\n",
    "            if batch % 100 == 0:\n",
    "                print('Epoch {} Batch {} Loss {} Accuracy {}'.format(epoch+1,batch,train_loss.result(),train_accuracy.result()))\n",
    "                checkpoint.save(file_prefix='./checkpoint/trf')\n",
    "        print('Epoch {}  Loss {} Accuracy {}'.format(epoch + 1, train_loss.result(), train_accuracy.result()))\n",
    "        print('Time take for 1 epoch:{} secs\\n'.format(time.time()-start))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:TF2.1] *",
   "language": "python",
   "name": "conda-env-TF2.1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}