03.01.01.image_classification_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Classification with CNN  \n",
    "\n",
    "이번 실습에서는 CNN을 이용해 MNIST 데이터셋의 필기체 숫자들을 분류하는 image classification task를 다루어 보도록 하겠습니다.  \n",
    "\n",
    "이번 실습을 통해 CNN이 추출해 내는 Feature가 classification task가 어떻게 규정되느냐에 따라 어떤 방식으로 다르게 추출되는지를 확인해 보고자 합니다.  \n",
    "\n",
    "(참고)  \n",
    "https://github.com/gilbutITbook/006975/blob/master/5.1-introduction-to-convnets.ipynb  \n",
    "https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6  \n",
    "https://www.kaggle.com/adityaecdrid/mnist-with-keras-for-beginners-99457  \n",
    "https://3months.tistory.com/150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, Activation\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "import json\n",
    "\n",
    "import matplotlib.pyplot as plt #for plotting\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare MNIST Dataset for CNN   \n",
    "\n",
    "keras에서 제공하는 MNIST 데이터셋을 로딩해 봅시다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to reproduce same training result\n",
    "from numpy.random import seed\n",
    "seed(1)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54000, 28, 28)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54000, 28, 28)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,  19,  50,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0, 153, 247,  57,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,  14, 199, 195,  48,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,  70, 145, 253, 153,  76,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,  59, 253, 185,  34,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,  70,  14,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,  23, 255, 254,  78,  16,  23,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0, 182, 148,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   4, 185, 254, 191,  12,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,  55, 240, 253,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,  70, 253, 254,  68,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  34, 247, 253, 200,   0,   0,   0,\n",
       "          0,   0,   0,   0,  73, 244, 253,  95,   3,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,  83, 222, 253, 253,  80,   0,   0,   0,\n",
       "          0,   0,   0,  16, 213, 253, 177,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0, 255, 254, 254,  95,   0,   0,   5,  59,\n",
       "         59, 134, 156, 201, 254, 254, 178,  23,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0, 254, 253, 253, 217, 214, 215, 217, 253,\n",
       "        253, 253, 254, 253, 253, 250, 233,  53,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0, 155, 253, 253, 253, 253, 254, 253, 253,\n",
       "        253, 253, 254, 253, 228, 111,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   5, 102, 117, 117, 117, 117,  34,  19,\n",
       "         19, 161, 254, 234,  73,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0, 157, 254, 211,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         62, 254, 218,  30,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "        182, 253, 128,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  31,\n",
       "        241, 253,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  61,\n",
       "        212,  64,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0]], dtype=uint8)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABjwAAAGNCAYAAAClqNhVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd7hlVXk/8O9iZpihKkWKSgdFsNAUhRjBihIVSwSjiBWNYsEW1GjUGBsq9o6CsaGxoLGLiPgDVEBFERSkK4IUpc7AzKzfH+eaiIHZ6957zj133/v5PM95ptzvXvtlEt97zn33XrvUWgMAAAAAANBnq427AAAAAAAAgOky8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADovYXjLgAAAABgvimllCRbJLlbkjsl2TTJekmWJFmcZHmSpUmuS3JZkkuTnJvk7FrrjeOoGQBmu1JrHXcNADAnlVK2TrJLku2T3DWDD7SbJNkwyRpJVk9yU5Ibk1yTwYfY3yc5O8kZSU6rtZ4785UDADAKpZR7Jnlkkr2S3DfJ2lNYZmWSs5J8P8l3k3yz1rp0SCUCQK8ZeDArlVJelOT2kzjkglrrUSMqB6DTxBV6uyTZO4MPsPdLsv4Qlr4oybeTfDbJ8bXWlUNYE2AoSil7JTl+3HXchtfVWl877iIASilrJ3lmkmcl2WEEp7gmyeeTHFFrPXME6wOMTCllcZJ7Jtlt4rVrkh3TtjPR3rXW74+uOvrIllbMOqWUg5IcMcnDTkhy1PCrAbhtpZQ1kzwiyT8keXiSjUZwms0z+ID8zCQXl1LeneSDtdbrRnAuAACGpJSyepJDkxyWyV3QN1nrJnlGkqeXUo5N8tJa629HeD6AKZnoi3fP/w43dpv486Jx1sXcYuDBrFJK2TjJO8ZdB8BtmXiDtm+SAyZ+XWsGT79ZksOTvKKU8pokH3DHBwDA7FNK2T3JJ5NsO5OnTbJfkoeXUl6f5M3eKwLjUkpZmMEwY9f873Djnhls7QwjY+DBbPP+DGcLGIChKqWsk+RNGQw6NhhzOesneW+Sp5VSnlhrPWfM9QAAMKGU8sIMLlIZ1xXLi5P8R5IHllIeX2v905jqAOaZie2p3p7BcONeSZaMtyLmo9XGXQD8RSnl8UkeO+46AG7Dxkmel/EPO/7arklOK6U8btyFAACQlFLekuSdmR3bszwoyQkTOykAzIQ1MvjcvHsMOxgTAw9mhVLKX65WBmBy1knyuVLKs8ddCADAfFZKeXWSl4+7jr9xzyTfmHhwOgDMeba0YrY4IoOrpwGYvNWSfLCUsrzWeuS4iwEAmG9KKY9O8vopHHpjkuOSnDzx+l2Sq5P8KYMrpdfL4A7jeyXZI8kDk2w9yXPsnOSoJI+fQn0A0CsGHoxdKWWfJE8Zdx0AI/KHJD+ZeJ2d5IIklyS5LskNGdyhscHEa5ckD0iyV5JNpnCuD5ZSzq+1fm/aVQMA0KSUcockH5nkYZcleXeSD9Var7yNzM1JrklyYZLTk3y8lFKSPCLJizMYfrR6XCnlybXWT06yTgDolVJrHXcNzGMTDwH+ZZLNp7nUCbXWvaZfEcCtK6Vsm6Tl4eA1yYlJvprkm7XWX07hXAsyuALvZRk8p2Myfp/k7rXWqyd7XoDJKqXsleT4jpj3acCcVkp5T5JDJnHIV5M8vdZ6xTTP+8wMnheyVuMhlybZptZ643TOC3BbSim3z+AutZmyd631+zN4PnrAMzwYt7dk1cOOk2aqEIBpOjvJS5NsXmt9QK31bVMZdiRJrXVFrfWYWutuSQ7K4G6QVndM8rapnBcAgMkppWya5OBJHPK2WuujpjvsSJJa60eT3C/tP1zcNInnvgGzTc3g4sLPZvCZeu8kbx9rRfSagQdjU0r5+yTPWUXk6gxu0wWYrWqSLyd5cK31brXWt9daLxnqCWr9RAZ3eUxmePLUUsrdhlkHAAC36qlJVm/MfjZDfqh5rfUXSR6dZGnjIc8a5vkBJqkmOTfJMRn0wwcmWa/Wepda6xMnPlN/P5O76A9uwTM8GItSyhpJPpqkrCL20gz2NQWYbZYn+XSSN9Vazx71yWqtv5nYNuaEJDs2HLJaklfE85EAAEbtgMbcH5M8q45gX/Fa64mllLck+beG+A6llHtMDEoARu28JKcmOW3i19NrrX8ab0nMde7wYFxen2S7VXz9+Frrx2aqGIBGNYMr87avtR40E8OO/znx4GGWj0rS+ubw8aWUdUdYEgDAvDbxsPJ7NMb/o9Y6yiuW35bk8sbsg0ZYBzC/LUtyWJKHJFm/1rpNrXX/Wutba63fM+xgJhh4MONKKfdOcugqIktjX1Fg9rkyya4Tt9n+dhwF1FrPy+BB5i3WSPLIEZYDADDf3Tur3rXgL5ZlsMPByEwMU1ovGtx9lLUA81et9cZa61tqrd+ttc7kw8vhfxh4MKNKKYuSHJlkwSpi/15rPWeGSgJoUmu9utb603HXkcEH2dY7Sx44ykIAAOa5uzTmflhrvX6klQx8qzHXWjcA9I6BBzPtlVn1Lb+/SHL4DNUC0Du11pVJPtgY33OUtQAAzHN3asydMNIq/tf/S3JTQ26zURcCAONi4MGMKaXcPYOBx21ZmcFD3G6eoZIA+uqLjbmtSykLR1oJAMD8tU5j7ncjrWLCxGfpPzZE1x51LQAwLgYezIhSyoIMtrJafRWx99VafzRDJQH0Vq314iQXN0QXxRV8AACj0vozlStGWsUttQw8XBADwJxl4MFMOTTJfVbx9YuTvGqGagGYC37TmLvdSKsAAJi/Wh/Iu3SkVUz+XB4kDMCcZeDByJVStk3y+o7Y82qt185EPQBzxJWNubVGWgUAwPx1VWNuw5FWcUt3aMi0vo8EgN4x8GCkSiklyUeTrLGK2OdrrV+doZIA5orlQ84BADA5FzbmWoYQw9JyrpatUQGglww8GLXnJHnAKr7+pyQvmKFaAOaS9Rpz1420CgCA+ev/NeZ2GWkVE0op2yVZtyHaWjcA9I6BByNTStksyVs6Yi+rtf5hJuoBmGO2bMzpsQAAI1BrvThtd3k8dGL3g1F7WGPuxJFWAQBjZODBKH0oyTqr+PoPkhw5Q7UAzBmllLWS3LUhenWt1R7NAACj8/mGzCZJ7j/qQpLs35C5PMlJoy4EAMbFwIORKKUcmOThq4gsS3JwrbXOUEkAc8mD0/Y9/IxRFwIAMM+9N8mKhtwbRllEKWXfJH/XEP1grXXZKGsBgHFaOO4CmHtKKRsneWdH7A211l/PRD0Ac9DjG3MnjLQKgEkopWySZM8kuyXZMYOt+TZNslaSxUmWJrkxybVJfpfkkiTnJjk9yem11gtmvGiADrXWC0spn09yQEf0/qWUJ9VaPzXsGkop6yZ5W0P0hiTvH/b5AWA2MfBgFN6bZP1VfP3MdD/bA4BbUUrZKMk/Nsa/NcpaABptX0o5PclOSVa1h/2aE68NcivPKSqlnJ/kG0m+lOQ4dwoDs8ihSR6aVX8OTpIjSykX1lp/OKwTl1JWT/LFJNs3xF9Ra71sWOcGgNnIllYMVSnlsVn1lccrkzyr1nrzDJUEMNe8IIMrobucX2u1PzMwG2ycZOesetjRYqskz03ynSS/LaUcVkpZ1fPiAGZErfUPSZ7XEF2c5KsTn5unbeLOua8leVBD/IQk7xnGeQFgNjPwYGhKKesleV9H7AO11pNnoh6AuaaUcucMriBs8bFR1gIwZlsleVOS80spLyulLBp3QcD8Vmv9bJJXN0Rvn+QLpZSPllI2m8q5SimLJp6beUYGz3brclaSJ7gzDoD5wMCDYToiySar+PrvkrxyhmoBmIsOz2C7ly7XpnsADTAXbJDkrUlOK6XsMu5igPmt1vqGJP/eGH9GkvNKKceUUh43cbfGbSqlLC6l7FFKeU2SC5J8IskdGs5zbpIH11ovb6wLAHrNMzwYilLKw5Ic1BF7Xq31mpmoB2CuKaU8Lt0Pw/yLd9darx5lPQCzzD2S/KiU8oJa6wfGXQwwf9VaX1NKOTPJBzO4m2NVFiZ5wsQrpZQLM7hQ8Ookf0qyJMl6GQx3t0/btqZ/7StJnl5rvXKSxwFAbxl4MG2llLWTfLgj9oVa67EzUQ/AXDNxxd8HG+OXZLDNC8B8szDJ+0spOyR5ga1bgHGptR5TSjklybuTPGoSh24x8Zquq5K8utb6/iGsBQC9YksrhuEtSTZfxdf/nOT5M1QLwJxSSlmQ5LNJNmw85CW11utHWBLAbHdIEnd5AGNVa72w1vroJLsk+WKS5TNw2iuT/GuSLQ07AJiv3OHBtJRS/i7JP3fE/qXWeulM1AMwB705yQMas1+stX5ulMUATNKfM3io7pkTr18l+WOSa/7qtTDJGhk8C+7OSe6eZNckD0qy6RTP++xSylW1Vs+PA8aq1vrTJI8rpayXwd0eL0yy8xBPcUOST2UwVDmu1nrzENcGgN4x8GDKSilLkhyZpKwi9sN0b3cFwK0opTwpyUsb439I8uwRlgPQoib5WZJvJPlmkpNrrV1XNa9IsiyD/erPTvLdv3yhlLJ7Bg/2PTCDvewn47BSyo9sqwqMWylllySPzWDgcY8hL79mkgcmuS7J0lLKD2qtK4d8DgDoDVtaMR2vS3KXVXz9piQH2z8ZYPJKKXtmMFRusTLJQbXWK0ZYEsCqXJrkjUm2rrXuUmt9Va31xIZhxyrVWn9Uaz04ydZJPpLBQKVVSXLUxHOQAGZcKeXBpZTvJTktyasy/GHHX2yT5NAkxyc5u5TyjFLKohGdCwBmNQMPpqSUsmuSl3TE3lhrPWsm6gGYS0op2yb5cpLFjYe8ttb67RGWBHBbLk7ymCSbTww5LhjFSWqtl04MPvbOYLjS6vZJ3jaKmgBuSyllm1LKiUm+k0HfmknbJflokvNKKY+Z4XMDwNgZeDBpE1eKfCzJglXEzkryppmpCGDuKKVsnME2MK0PKf96kjeMriKA21Zr/W2t9cvTvZNjEuc7IcluSX4+icOeVEq5z4hKAriFUsrTM9ja7+/GXMqdk3yxlPLRUsraY64FAGaMgQdT8Yok91zF12uSZ9Vab5qhegDmhFLK7TIYdmzTeMjPkjzR1oHAfFJr/X0GDzQ/cxKH/cuIygH4H6WUd2SwJelkBgxXJflKkpcleUCSHZJskmT1DO5S2zqDQe8BSd6X5IxMbnu/ZyQ5pZSy0SSOAYDe8tByJqWUsmMGe4+uyodqrf9vJuoBmCtKKWsm+VqSnRoPOSfJw2qt14yuKoDZqdZ6ZSnl0UlOzeAHgl32K6VsOaottwBKKYdn8ByNVmdmsOXep1dxseCfJ17nZ/AckGMmzrV9kpcmeXLatkDdMcn3Sil711r/OIkaAaB33OFBs1LKahlcrbL6KmK/T3LYzFQEMDeUUhZn8MyOPRsPuTjJg2utl4+uKoDZrdb62wx+4NditST7j7AcYB4rpbw47f1oaZLnJLlHrfWoqeyMUGs9u9b6zAye1/HDxsN2TPLtifedADBnGXgwGYcm2b0j8/xa659nohiAuaCUsjDJ55I8pPGQyzIYdlw0uqoAeuNjGVz13OLxoywEmJ9KKTskeWNj/LdJdq+1fmgYW5LWWi9OsleSNzceslM8+w2AOc7AgyallG2SvL4j9uVa6xdnoh6AuaCUsiDJp5I8qvGQq5I8pNb6m9FVBdAfEz8wbP1B3y4Tz0oCGIqJ93JHp21bqcszeB93xjBrqLWuqLW+IsnbGw95cSnl/sOsAQBmEwMPOpVSSpKPJFlzFbFrkhwyMxUB9N/ENoEfT/KExkOuyeCZHb8YXVUAvXRskqsbcqslue+IawHml8dk8EDxLsuS7FdrPX+Etbw8g37YZbW0D4oBoHcMPGhxcJK9OzKvqLX+biaKAei7iUHyh5Mc2HjI9UkeUWs9dXRVAfRTrfXmJF9rjO88ylqAeedFjbn31VpPHmUhtdaVGTwb5IaG+B6llK7tqgGglww8WKVSyp2TvLUjdlKSD8xAOQBzxfuSPKMxuzTJo2qt/2+E9QD03YmNua1GWgUwb5RSdk6yZ0P0uszQHRW11j8keW9jvHVYAwC9YuBBlw8mWXcVX78pycHDeOAawHxQSnlXkn9ujN+U5HG11u+NsCSAueDMxtyWoywCmFf2acwdXWv940gruaW3J2n5fL7PxBarADCn+ObGbZp4qOO+HbG31FpbP2ACzGullLcleUFjfHmS/WutXx9hSQBzReu++OuNtApgPml98Pc3R1rF36i1Xp7kpw3R2ye554jLAYAZt3DcBTCrlYbMdqWUo0ZYw9qNue0nUcdza60t+5oCDE0p5U1JXtIYX5HkybXWL4+wJIC55M+NubVGWgUwL0w8j22PhujNSb4/2mpu1XeS7NKQu3+Sn424FgCYUQYeTNcB4y5gwsZJDmrMvihtD3IDGIpSyuuTHNYYr0meXms9ZoQlAcw1SxtzS0ZaBTBf3G7i1eXcWut1oy7mVrQOMbYcZREAMA62tAKAESqlvCbJqxvjNcmza62fGGFJAHNR6yCjdTACsCobNOauGGkVt631mSGt/x0A0BsGHgAwIqWUw5K8bhKHPL/W+pFR1QMwh7VcaZ0k14+0CmC+aB0UzOTDyqdyXgMPAOYcAw8AGIFSykuTvGkSh7yk1vq+UdUDMMdt1Zj700irAOaLRY25ZSOt4ra1buFsmz8A5hwDDwAYslLKC5McPolDXllrfceo6gGYB3ZszF0wyiKAeePGxtyGI63itq3ZmGv97wCA3jDwAIAhKqU8N8k7J3HIa2utk7kTBID/6+8ac+ePtApgvmjdHm9cA487NubG8UB1ABgpAw8AGJJSyrOSvHcSh7yp1jqZZ3wA8DdKKYuS7NsYP2OUtQDzxiWNuS1KKWWkldy67RpzvxtpFQAwBgvHXQCzV631T0nG8ebsf5RStkzblXgn1Fr3GmkxAKtQSnlqkg+lvW++o9b6ytFVBDBvPDLJ+g25muTkEdcCzAO11utLKVek+w6O9ZPsnOT00Vd1C/drzF0wyiIAYBzc4QEA01RKeVKSI9M+7HhPrfUlIywJYD45rDH3i1rrVSOtBJhPzmzMPWSkVfyNibve9mmM/2qUtQDAOBh4AMA0lFKekOTotH9P/XCSF46uIoD5Y+Luuns3xv9rhKUA80/rHWMHllJm8mcvj0myXkNuRZIfj7gWAJhxBh4AMEWllP2SfCrJgsZDPp7kObXWOrqqAOaHUspWSd4xiUOOGVUtwLz0/cbcjkmeOMI6/sfEYOUVjfFTa62tD18HgN4w8ACAKSil7JvBD89an4f1qSTPNOwAmL5SynpJjk3bVcxJ8q1a629GWBIw/3wvSes2ea8rpaw9ymImHJJkp8bsF0ZZCACMi4EHAExSKeUhGXxIXL3xkGOSHFRrXTm6qgBmRillt1LKY0sprc8tGvb5N05yXJJ7TOKwN46oHGCeqrXenORzjfFtkhxTSmm9K3jSSin3TfLWxviKuOsNgDnKwAMAJqGU8oAMripe3HjIF5M8uda6YnRVAcyoO2cw9D29lPJPEw/InRGllD2TnJpk50kc9tVa6w9GVBIwvx2RpPWClkckef8ohh6llN2TfD3t708/V2u9aNh1AMBsYOABAI1KKXsk+e8kazQe8pUkB9Ral4+uKoCx2SmD7fouKqW8sZSyw6hOVErZuJTyviQ/yGDg0ur6DLZ4ARi6ia3yJnOnxMFJvldKueOwaiilvCCD3ti6xd+KuOsNgDnMwAMAGpRSds3gyrnW/Ze/keQfJ7Y7AJjLNsngIblnllLOKKW8qZTygFLKkukuPLF91vuTnJfkuZn855d/dhUzMGIvTXLNJPJ/n+TnpZSXllLWnepJSyl7lVK+k+Rdad9mNUneXWv95VTPCwCzXfHsVGazUsqWSc5viJ5Qa91rpMUA81op5agkB427jik4utb61HEXAcwdpZT9knypIXpzkp8n+WmS3yQ5J8mlSS5LcnWSZUluSrIogzvnNkqyWZIdk+ya5MFJ7jSNUo+otb54GscDNCmlHJTkqCkcek2ST2TwXKITa61XruIcC5LsksHA5IAku03hfGcluU+t9bopHAvQrEefn59Waz1q3EUwXAvHXQAAADAnLcrgB3JT+aHcdH0sg6uuAUau1np0KeXeSZ43yUPXzWDbvUOS1FLK+UkuT3JVkj9lMAxef+K1VdrvNL41f06yn2EHAHOdgQcAADCXvC/J86tb2YGZ9YIk6yR5yhSPL0m2nngN25+TPGzimSMAMKd5hgcAADAXLE3yjFrrIYYdwEyrta5M8rQk7xx3LX/j4iQPrLX+aNyFAMBMMPAAAAD67sQku9ZaPzbuQoD5q9a6stZ6aJInZrAl1bh9I8kutdbTx10IAMwUAw8AAGAyZtPdE+dk8EDMB9RafzXuYgCSpNb62STbJ/nPjKdnXpLkgFrrI2qtV4zh/AAwNgYeAADAZHw1gweRH5bku0lunOHz1yTHJzkgyfa11k/YwgqYbWqtl9Van5JkxyRHZbDt3qidneTgJNvUWo+ZgfMBwKzjoeUAAECziX3qT5t4vaWUsjjJHkl2T7JLkp2TbJPBA3iH5foMtq36dpL/qrVePMS1AUam1npWkqeVUl6Y5HFJ9kvygCS3G9IpzkzyrSSfrbX+ZEhrAkBvFRdDAQAAw1RKWTfJPZNsnWTzJFtM/LppkrUmXmsmWSODwcjSidefk1w68To/yS+T/CLJL2utN8/sfwXAaJRSFiTZKcm9ktw9gyHxpkk2TrJukiVJFidZkUFvvDHJFRn0xkuSnJVBf/xxrfXyma4fAGYzAw8AAAAAAKD3PMMDAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcWzuTJVi+L65KsNZOnBHpsaa7PTXVZGXcdw6YXApN1ba6+otZ6h3HXMUx6ITBZc7EXJvohMDk+JwOsuhfO6MBjSdbK7uVBM3lKoMd+VI8bdwkjoRcCk/Xd+l8XjruGYdMLgcmai70w0Q+ByfE5GWDVvXBaW1qVUvYppfy6lHJuKeWw6awF0Gf6IYBeCJDohQCJXgiMz5QHHqWUBUnel+ThSXZI8sRSyg7DKgygL/RDAL0QINELARK9EBiv6dzhcZ8k59Zaz6u13pTks0kePZyyAHpFPwTQCwESvRAg0QuBMZrOwONOSS7+qz9fMvF3t1BKObiUcmop5dSbs2wapwOYtTr7oV4IzAN6IYDPyQCJXgiM0XQGHrf2FPT6f/6i1g/XWnerte62KIuncTqAWauzH+qFwDygFwL4nAyQ6IXAGE1n4HFJks3+6s93TvL76ZUD0Ev6IYBeCJDohQCJXgiM0XQGHj9Jsl0pZatSyupJDkjyleGUBdAr+iGAXgiQ6IUAiV4IjNHCqR5Ya11eSjkkybeSLEjysVrrmUOrDKAn9EMAvRAg0QsBEr0QGK8pDzySpNb69SRfH1ItAL2lHwLohQCJXgiQ6IXA+ExnSysAAAAAAIBZwcADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN5bOO4CAGA2WrH3Lk255Wsu6Mws+e4ZTWvVZcuacgAAAAD8X+7wAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6b+G4CwCA2egx7/tOU+7g213QmfnXy3dtWut7775fU279j5/clAMAAACYT9zhAQAAAAAA9J6BBwAAAAAA0HsGHgAAAAAAQO8ZeAAAAAAAAL1n4AEAAAAAAPSegQcAAAAAANB7Bh4AAAAAAEDvGXgAAAAAAAC9Z+ABAAAAAAD03sJxFwBMzcI73bEpt+wum3RmFhx/+nTLgTnn3V/6h6bcwU99b2fmDRud1nbSN7Tldtng+Z2ZO77tpLZzAgAAAJN2zrt378yc9/gPNa217fFP68xs86SfNq0137nDAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6b+G4CwCm5jeHb9SUO/n+7+nMPP7gFzWttfgbP2nKwVyw5at/3JTb85xDOjPfe8MRTWstLouacj980ds7M489vbuuJFn4vdOacgAAo1YWtv2I4rpH79qUu+xxyzoz66x9Y9NaP9n1M025Qy/dvTOz3qIbmtb6+Z/u3JQ754oNOzNrH7tu01obfOPcptyKP/6xKQfQRws2bvuZ2wH3P7kzc3Nd0bTWMXt8qDPzivs9q2mtcvLPm3JzlTs8AAAAAACA3pvWHR6llAuSXJtkRZLltdbdhlEUQN/ohwB6IUCiFwIkeiEwPsPY0mrvWusVQ1gHoO/0QwC9ECDRCwESvRAYA1taAQAAAAAAvTfdgUdN8u1SymmllINvLVBKObiUcmop5dSb0/3AMICeWmU/1AuBeUIvBPA5GSDRC4Exme6WVnvWWn9fStkoyXdKKWfXWn/w14Fa64eTfDhJ1i3r12meD2C2WmU/1AuBeUIvBPA5GSDRC4ExmdYdHrXW30/8enmSLyW5zzCKAugb/RBALwRI9EKARC8ExmfKA49SylqllHX+8vskD03yy2EVBtAX+iGAXgiQ6IUAiV4IjNd0trTaOMmXSil/WefTtdZvDqUqgH7RDwH0QoBELwRI9EJgjKY88Ki1npfkXkOsBUiyYMMNmnL73rXt4ohXXfrgzsyiG5Y3rcWt0w/nqJUrmmLrHXVyZ2avRYc2rXXya9/blFu7LO7MHPj+rzat9bnHPKApt+Ksc5pyzPLF7D4AACAASURBVF96IYBeuCoL73ynzszZb96oaa2z935/U+7qlUs7M48988Cmtbb/5POacsN0v78/syn3wh2O78zsussFTWtd+Zq1mnLP+frTOzN3O+IPTWstP//Cphz9oRfSd5see0NT7nUb/XRo57zn6gs6M9dtvkbTWut0/5hiTpvWMzwAAAAAAABmAwMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHpv4bgLgPnk2v3v25m5dJ+bm9Y6dpMPN+V2OOEZnZn1N1/StNbtm1LA39rgIyc35e6x2SFNubOf+YHOzIHr/KFprTf+2xpNua0PWtyZqcuWNa0FAMw/1+56p87M2Xt3v8dJkk9du2lT7j+f98jOzFrHnda01tY5ryk3TJc15r6QjTozX77rHk1r/fo5GzbltrrH7zszB37zxKa1jnryP3Rm6k9+0bQWwKos2G7rptwLN/l044qLpl7M39j/t/t0Ztb7wQVNay2fZi195w4PAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3Fo67AJhPVr9uRWfmNw/9cNNa+/92n6bctm9e2plZecbPmtYCRmuL15zclHvZP+zcmXnjxqc2rXXm/T/elNv5Rc/vzNzpLSc1rQXAcJVFqzflbnzYTp2Z6zdd0LTWVTutbMot/mP3epu/1veP+eCSB5WhrfWWzz6+Kbf5cf5/6y9W/Prcpty2h7blWnz8uD2bco85+nudmf963sOa1lpw/OlNOWDuWXCXbTozu3/+rKa17rZo0XTL+R+XrrixKXfVm7fszCy+9CfTrGZ+cIcHAAAAAADQewYeAAAAAABA7xl4AAAAAAAAvWfgAQAAAAAA9J6BBwAAAAAA0HsGHgAAAAAAQO8ZeAAAAAAAAL1n4AEAAAAAAPSegQcAAAAAANB7C8ddAKt27f73bcrd/rhzOjMrrrhyuuVwGxZuuXlT7lsf+UhnZkUtTWtdcuS2TbkNL/ttZ6a1/uUXXNSUA0brjOfcvTNz0edPbFpr84VrNOW2fUR3L7nxLU1LAT2yYIP1m3K/efcWnZlt3rW87Zzn/6Epd92eW3Vmrtl8uB931vjjys7MjXdou6asPvDqzswW63VnkuT1mx/blPvU1Td3Zr510fZNay05db2m3J2+f0NTjrlv41O6P+dc8ugbm9Y68RmHN+X2P/EFnZmFx53WtBaTV/a5vCl37U+XdGbOf+TqTWtte3xTDJiDfv2cO3Rmjt3wczNQyS2dvmyTptzir/9kxJXMH+7wAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeWzjuAli19f/5wqbcr/bdojOz3VOunG4581JZtHpnZvEnbmxaa0Vd2Zl52kV7Na214VfObjvn1Vc35YAe+fEvOiPvuWKvpqUO3+RHTblPbnNsZ+ZRDz2kaa1F3z61KQeM3++ftH1T7ty939+Zudepz21aq95/nabc/fb/aWfmDqtf17TWMH3q1N2bchsee/vOzLW/X6tprVdc9rSm3Mqf/aozs2nOaloLJmvdT5/Smdl/0cua1vrvN7ytKXf0x9/VmdnrmLZzbvfq7p6TJCuXLm3KzQf15puacp888mHdoc26P0snyZ+ffN+m3JKrV3RmFn/tJ01rAaO1YMe7NuXe/shPjriS/+uKFd0/D3zni57TtNbi6DnD4g4PAAAAAACg9zoHHqWUj5VSLi+l/PKv/m79Usp3SinnTPy63mjLBBg//RBALwRI9EKARC8EZqeWOzyOSrLP3/zdYUmOq7Vul+S4iT8DzHVHRT8EOCp6IcBR0QsBjopeCMwynQOPWusPklz1N3/96CRHT/z+6CT7DbkugFlHPwTQCwESvRAg0QuB2Wmqz/DYuNZ6aZJM/LrRbQVLKQeXUk4tpZx6c5ZN8XQAs1ZTP9QLgTlOLwTwORkg0QuBMRv5Q8trrR+ute5Wa91tURaP+nQAs5JeCKAXAvyFfgigFwKjMdWBx2WllE2TZOLXy4dXEkCv6IcAeiFAohcCJHohMGZTHXh8JclBE78/KMmxwykHoHf0QwC9ECDRCwESvRAYs86BRynlM0lOTnLXUsolpZRnJHlzkoeUUs5J8pCJPwPMafohgF4IkOiFAIleCMxOC7sCtdYn3saXHjTkWuaVmx62W1PuPVu9qym31XZLOjP/kF2b1uKWrnnsLp2Zr27zvsbVSmfit++4W9NKa1/9o8ZzMiz6IX3y9W/duyl3+EFtveTrN2zcmVly+vlNa61oSjFb6YVzQ1nY+TFgkHvQVU25m2v3/7Lv8NOlTWst+P7pTbkL3t6QyYKmtYbpLjl1xs+5csbPiF44GusdfXJT7p8uen5T7iHvPrEzc9Y/vbdprT13OKApt8Eru/vryp+f1bTWfHHNPW7qzDz2Xm3fG173hJOacvf8wos6M9t9rWmpeU0vZDrKotWbcks+0PZ+dN81/zydcqbk/se8rDOzzdfavrcxPCN/aDkAAAAAAMCoGXgAAAAAAAC9Z+ABAAAAAAD0noEHAAAAAADQewYeAAAAAABA7xl4AAAAAAAAvWfgAQAAAAAA9J6BBwAAAAAA0HsGHgAAAAAAQO8tHHcBc9Fq99y+M/P5j76raa3brbakKXfYH+7dkFrZtNZ8cc0/3bcp9/k3v60htUbTWrscfkhnZtMv/bhprdqUAuarF+/3labcailNuX89bb/OzFZX/LxpLWD8/vyE3ZpyJ+76zqbcff7jxZ2Zjb5/UtNaAF0WHH96U+743dbvzHzm2c9vWuuoQ49oym3x3ys6M3u/9aVNa238ntnbN8uuO3ZmLnlV21pn3uf9nZlFZUHTWi+59P5Nubu99aLOzPKmlYCpunGfnZpyX93mAyOu5P8646buXp4kW/330hFXwlS4wwMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAem/huAvokyufeb+m3Mtf/unOzO1WWzLdcm7hB+/dvTOzfk4e6jlnqwUbb9SU+6dXfaMpt/GCNToz+5z1mKa1NnnnSZ2Z2rQSMJ+VxYs7M4vKiqa1VjZ2nYW/WqspB8wC97lHZ+R1rz+yaam1G9+z/te/vLUz8/C/f27TWtv9y9VNueUXXdIdqt5ZwXxWly3rzGz87u7PaEnyyq8f0JRbeOSNnZmXHXJM01pv2GD/ptzWR17UmalLu/8tkuTCZ23XlPvSsw/vzKzfeIntZ67dpjPzH9/er2mtu721+98iSZb/7vdNOWBqFm6ycWfmJe/45AxUMjUvOeR5TbnFJ/xkxJUwFe7wAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcMPAAAAAAAgN4z8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6b+G4C+iTU1//gabcirqyIVWa1trl8EOacpt8/KSmXIuyeHFTbumD7zm0c170iLbZ26INb+zMPGq7XzSt9dzbn9+Ua/m/1WqlNq20cMvNOzPLL7ioaS1g/trsB93fvp+y7u+a1trjp09sym1x+OmdmZbvfsDULbjbdk25L33pY52Zk5e2vd+778ue05RbbXl3ZskT/9y01sd/+Nmm3B6fe0lnZpuXnNK0FkCXFee2fX5c8YDuzCfuu2/TWnu884ym3Auf+t3OzK+Wbdq01uPW/mZT7mkXdv83XPzGuzStteS/f9yZ2S5t/bzh2xEwA67bbYvOzMPX/PoMVHJLX7vhdk25NU+9sCm3YjrFMDLu8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAem/huAvokxV1ZVNuZerQzvmCZ3+xKbf04EWdmQVpq39RWdGUe8q6JzXlhmm1lM5M679/279Gsqze3Jn5962+3LTWK7c9uDOz6IKLmtYC+qPsvGNT7oY339CU++Cdv9CZuWj5jU1rbfivbW8FVi5d2pSbrVZbc83OzHmvuFfTWlu++uTplgNTcs2O6zflFpfu94VP/+azmtba7lOnNOVarHNMW+7A+zynKfevn+h+n/wfKx7XtNbWL/e/a2DmLLh2WVPuzKs2acrdc/MlnZkdF13ZtNY//nafptz1f//HzsyS/LhpLaA/Vuy9S1PuXe95T0NquD+WPvCCh3Rmrjnodk1rrbjs/OmWwxh13uFRSvlYKeXyUsov/+rvXltK+V0p5WcTr0eMtkyA8dMPAfRCgEQvBEj0QmB2atnS6qgktzbiP6LWutPE6+vDLQtgVjoq+iHAUdELAY6KXghwVPRCYJbpHHjUWn+Q5KoZqAVgVtMPAfRCgEQvBEj0QmB2ms5Dyw8ppZwxcfvaekOrCKB/9EMAvRAg0QsBEr0QGKOpDjw+kGSbJDsluTTJ228rWEo5uJRyainl1JvT9mAugB5p6od6ITDH6YUAPicDJHohMGZTGnjUWi+rta6ota5M8pEk91lF9sO11t1qrbstyuKp1gkwK7X2Q70QmMv0QgCfkwESvRAYvykNPEopm/7VHx+T5JfDKQegX/RDAL0QINELARK9EBi/hV2BUspnkuyVZMNSyiVJ/i3JXqWUnZLUJBckefYIawSYFfRDAL0QINELARK9EJidOgcetdYn3spfHzmCWma9u5341Kbcmff/+NDO+ZR1fze0tVZLacqtTB3aOWez429c0pR74X8+vzOz+etOalprUU5ryjE76YfcmgU73KUzc8BnvtW01hPXuazxrN39/KGfeVnTSlv/7OTGc86s1dZaqyn3u+fcqyn39wd099+v3PG9TWv9w6t3bcrNVXrh+Kz1hR835fb58m6dme1WtK01Fj/+RVPss/faqjPz+bPf2bTWod87pCm3+jd/0pRj7tMLuS11j+73Juu/7cKmtY7d4rim3FMu3Lsz8y+btr0fvfOaf2rK/bopxVynF84/F+y7elNux9U7f+Q8dKee3P3ZfJtzT5mBShi3qT60HAAAAAAAYNYw8AAAAAAAAHrPwAMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3ls47gL6ZOuDft2U2/lFz+/M3LxunW45I7P1MVc35Va78pqhnfPC96zXlPv57v/ZmfnpTSub1vq31zyjKbf5p09qygH9sGDDDZpyvz5i86bcbx545HTK+RulKbXTEYd0ZrZ+W1vvuvjVezTlVtz9uqZci1/f/xPd56ttvTw5sSn1heu7v8/s+4S27wslP2vKwdDVtvePdfnyERcyO9Rlyzozj/+vFzWttWj3tv67+TebYsA8tuHhF3VmXnHHbzStdfePvbgpt+WrT+7MPO71bWud/vR3NeV2e1V3f93sjd11JWn+/gaMzk0P260pd9L+b29cccnUi/kbn7jmTk25u3zkis7MiukWQy+4wwMAAAAAAOg9Aw8AAAAAAKD3DDwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6L2F4y6gT+qyZU25O73lpBFXMlorh5xrsXz5BkNb64fX37Upt+6nTxnaOWE+WbDB+k25i561fWfmpnXrdMu5pa1u6Iy8dbcvNC2175rfbsoNsxe22u3xv+jOPP3KprX2WeutTbnNF67RlGvx42Xd/2ora9s1Gf/8iyc15e546NLOTDnvZ01rwbCtdq+7NeVW/vysEVfSM6WMuwJgHrni4Ps15b6wxRGdmYcd+qKmtbb8/MlNuRZb/+dlTbnfHNj2/vznz31PZ2a/j+3btNbyS//QlANG6KV/bIqtt9qSERfyf334jY9pyt3+18PrmfSbOzwAAAAAAIDeM/AAAAAAAAB6z8ADAAAAAADoPQMPAAAAAACg9ww8AAAAAACA3jPwAAAAAAAAes/AAwAAAAAA6D0DDwAAAAAAoPcWjrsA5rYbH32fptyZe36wKfef127SmfnuP+7atFZyTmMO+GsHn/Ljpty+a35nxJVMzWopTbmVI65jOj662QmdmZWpTWu95NK9m3LfveCunZm1v7pO01rrHX1yU67FRjm7Kbd8aGeE4VvnfZc35a55/o5NufrTM6dTztjV+92rKXflK2/szJyy09ua1nrMC1/clAPmr/X3v6Qp99/Xb9qZuf2JFzStNcz3LyvOOa8pd9j5j23KffWuX5lOOcAMWrbvvTszn7jrEY2rrTG9Yv7Kt29cqym3wSmXNeVWTKcY5hR3eAAAAAAAAL1n4AEAAAAAAPSegQcAAAAAANB7Bh4AAAAAAEDvGXgAAAAAAAC9Z+ABAAAAAAD0noEHAAAAAADQewYeAAAAAABA7xl4AAAAAAAAvbdw3AUwt6176MVNuZWpTbl//8ZjOzPbnnVK01rA1Lznwgc25f7urp/pzNxutSXTLecWvnT9+p2ZPy5ft2mto867X1Pump9v0JRrseWx1zXlFv7uyqGdc+WVVzXlNlv6y6GdE7ilCz90l6bcy4/5VFPuP85+eGfmT1et3bTWoktWb8otufufOjNP3vbHTWsdfPsPNuVe/vvu70cHPvJZTWut+bMfNeWAuWe1ddZpyj1s4zObcq/50gGdma3+cHLTWuPw6ws3aQvedbR1AN2WP2jXptxD3/SDzsymC9aYbjmT9oKvPLUpt+05fs7H5HTe4VFK2ayUcnwp5axSypmllBdO/P36pZTvlFLOmfh1vdGXCzAeeiHAgH4IoBcCJHohMDu1bGm1PMlLaq13S3LfJM8rpeyQ5LAkx9Vat0ty3MSfAeYqvRBgQD8E0AsBEr0QmIU6Bx611ktrradP/P7aJGcluVOSRyc5eiJ2dJL9RlUkwLjphQAD+iGAXgiQ6IXA7DSph5aXUrZMsnOSHyXZuNZ6aTJocEk2GnZxALORXggwoB8C6IUAiV4IzB7NA49SytpJvpDkRbXWayZx3MGllFNLKafenGVTqRFg1tALAQam0g/1QmCu8d4QQC8EZpemgUcpZVEGjetTtdYvTvz1ZaWUTSe+vmmSy2/t2Frrh2utu9Vad1uUxcOoGWAs9EKAgan2Q70QmEu8NwTQC4HZp3PgUUopSY5Mclat9R1/9aWvJDlo4vcHJTl2+OUBzA56IcCAfgigFwIkeiEwOy1syOyZ5MAkvyil/Gzi716Z5M1JPldKeUaSi5L842hKBJgV9EKAAf0QQC8ESPRCYBbqHHjUWn+YpNzGlx803HLokwU73KUz866tPta42hpNqW0PPaVxPRguvfB/LXzwRU25A/Z6fmdm+ZoLplvOLfz/9u4v1rLyLAP483YGqrQQaEsNQbQtaWNqLyiQthGjXEhtsRGptcqFQW9oojUlMURpTEqMJqYWNPGCpA3Eaup/aIXERkhs05IQ/o1jgU4opAEZOg4gNoCmwsx8Xsw+ZjI5e/aaOWfOt77T3y+ZzJl1vlnn2Xud75kzebP2Pu3+b69cc/D5/5x0rjfkWxPXbb0DHb4mrNGHm+/Mv7x30rpb7nrftBN+eEIz/fihSad69cyJ6/aesXLNzU9dNulc//zFn560bsdXd01Y9c1J54LjpQu3j9eccfqkdb911uOT1t363Q9sJM5Js+ON035qvf59X5607rr/eO/KNYf+67uTzsW4dGFf//PmUyatu+6NW//z0L++svpnyB+76elJ5/L/X47X5DctBwAAAAAAmCsDDwAAAAAAYHgGHgAAAAAAwPAMPAAAAAAAgOEZeAAAAAAAAMMz8AAAAAAAAIZn4AEAAAAAAAzPwAMAAAAAABiegQcAAAAAADC8nb0DMK7/Pv/MlWvufPldk8512+/97KR1p+W+SeuA/nZ8ddfqNZv8NQ9u8vkA5uTg/mcnrTv75tXrzt5oGIBtor3y6qR1j75yYNK6gxe9tJE4J2TH2atb/Yk/O2fSuX79jLsnrXvnHVesXPO279076VzAidlx9bSfDXv4/ad+fuWaA3uf2YIkfD9yhwcAAAAAADA8Aw8AAAAAAGB4Bh4AAAAAAMDwDDwAAAAAAIDhGXgAAAAAAADDM/AAAAAAAACGZ+ABAAAAAAAMz8ADAAAAAAAYnoEHAAAAAAAwvJ29AzCuH7jz/pVrvnznmZPOdVru22gcAAAAOC4Hn3tu0rqP/MO1k9b9yy//8co1t/zbeyed628fu3DSuhsv/PuVa37itS9MOteVj//ipHVv//RjK9ccnHQm4ET94B+cMWnd+z/14ZVr7nrn7ZPOdc3Tl05ad/DnXpy0Dk4Gd3gAAAAAAADDM/AAAAAAAACGZ+ABAAAAAAAMz8ADAAAAAAAYnoEHAAAAAAAwPAMPAAAAAABgeAYeAAAAAADA8Aw8AAAAAACA4e3sHQAAAABgzs6/7t5J637mwHUr13z0g/dMOtfDl/z5pHVf+96pK9dceOe1k871jt+4f9I6oL/X3LN70rpTL1u95kO5aOJXfWniOujHHR4AAAAAAMDwDDwAAAAAAIDhGXgAAAAAAADDM/AAAAAAAACGZ+ABAAAAAAAMz8ADAAAAAAAYnoEHAAAAAAAwPAMPAAAAAABgeAYeAAAAAADA8Hb2DgAAAACwHbz1+ntXrnng+h2TzvWhXLTROP/vHbl/084FAHPmDg8AAAAAAGB4KwceVXVeVX2lqvZU1aNV9YnF8Ruq6pmq2r34dfnJjwvQhy4E0IUAa/QhgC4E5mnKS1odSPLbrbVdVXV6koeq6u7F5/6ktfaZkxcPYDZ0IYAuBFijDwF0ITBDKwcerbV9SfYtPn6pqvYkOfdkBwOYE10IoAsB1uhDAF0IzNNxvYdHVb0lybuT3Lc49PGq+kZV3VpVZy35O9dU1YNV9eCr+d8NhQWYA10IoAsB1uhDAF0IzMfkgUdVvT7JbUmuba29mOTmJOcnuSCHp7k3rvf3Wmufba1d3Fq7+JS8dhMiA/SjCwF0IcAafQigC4F5mTTwqKpTcri4vtBauz1JWmv7W2sHW2uHknwuyXtOXkyA/nQhgC4EWKMPAXQhMD8rBx5VVUluSbKntXbTEcfPOWLZlUke2fx4APOgCwF0IcAafQigC4F5Wvmm5UkuSfKrSR6uqt2LY59MclVVXZCkJXkyycdOSkKAedCFALoQYI0+BNCFwAytHHi01u5JUut86p82Pw7APOlCAF0IsEYfAuhCYJ4mv2k5AAAAAADAXBl4AAAAAAAAwzPwAAAAAAAAhmfgAQAAAAAADM/AAwAAAAAAGJ6BBwAAAAAAMDwDDwAAAAAAYHgGHgAAAAAAwPAMPAAAAAAAgOEZeAAAAAAAAMMz8AAAAAAAAIZn4AEAAAAAAAzPwAMAAAAAABiegQcAAAAAADA8Aw8AAAAAAGB4Bh4AAAAAAMDwDDwAAAAAAIDhGXgAAAAAAADDq9ba1n2xqueSPHXU4TcleX7LQmw++fuSv6+Tnf9HW2tnn8Tzd6ELZ0n+vkbPn+jD46YLZ2n0/Mn4j0H+Y9t2XZjow5mSvy/5j00XjkP+vuTvq1sXbunAY90AVQ+21i7uGmID5O9L/r5Gzz8noz+X8vclf3/b4THMwejPo/z9jf4Y5GfN6M+l/H3J39fo+edk9OdS/r7k76tnfi9pBQAAAAAADM/AAwAAAAAAGN4cBh6f7R1gg+TvS/6+Rs8/J6M/l/L3JX9/2+ExzMHoz6P8/Y3+GORnzejPpfx9yd/X6PnnZPTnUv6+5O+rW/7u7+EBAAAAAACwUXO4wwMAAAAAAGBDDDwAAAAAAIDhdRt4VNUHquqxqnqiqn63V44TVVVPVtXDVbW7qh7snWeKqrq1qp6tqkeOOPaGqrq7qh5f/H5Wz4zHsiT/DVX1zOI67K6qy3tmPJaqOq+qvlJVe6rq0ar6xOL4ENfgGPmHuQZzpQ+3li7sSxeyjC7cWrqwL13IMrpw6+nDfnQhy4zehcl4fagL+9KHm5ynx3t4VNWOJN9KclmSvUkeSHJVa+2bWx7mBFXVk0kubq093zvLVFX1U0leTvIXrbV3LY59OskLrbU/WvwjclZr7Xd65lxmSf4bkrzcWvtMz2xTVNU5Sc5pre2qqtOTPJTkF5L8Wga4BsfI/9EMcg3mSB9uPV3Yly5kPbpw6+nCvnQh69GFfejDfnQh69kOXZiM14e6sC99uLl63eHxniRPtNa+3Vp7JcnfJLmiU5bvG621ryV54ajDVyT5/OLjz+fwN+MsLck/jNbavtbarsXHLyXZk+TcDHINjpGfjdGHW0wX9qULWUIXbjFd2JcuZAld2IE+7EcXsoQu7EAX9qUPN1evgce5SZ4+4s97M94/Ci3JXVX1UFVd0zvMBvxQa21fcvibM8mbO+c5ER+vqm8sbl+b5a1dR6uqtyR5d5L7MuA1OCp/MuA1mBF9OA/D7cN1DLcPdSFH0IXzMNw+XMdw+1AXcgRdOB/D7cV1DLUXdSFH2A5dmGyPPhxuL65juL2oDzeu18Cj1jm29a+ttTGXtNYuTPLBJL+5uHWKrXdzkvOTXJBkX5Ib+8ZZrapen+S2JNe21l7sned4rZN/uGswM/qQzTDcPtSFHEUXshmG24e6kKPoQjbLUHtRF3KU7dCFiT6cg+H2oj7cHL0GHnuTnHfEn384yXc6ZTkhrbXvLH5/NskXc/iWuxHtX7zO2trrrT3bOc9xaa3tb60dbK0daMaNHAAAAdBJREFUSvK5zPw6VNUpObzxv9Bau31xeJhrsF7+0a7BDOnDeRhmH65ntH2oC1mHLpyHYfbhekbbh7qQdejC+RhmL65npL2oC1nH8F2YbJs+HGYvrme0vagPN0+vgccDSd5eVW+tqlOT/EqSOzplOW5V9brFG7Ckql6X5P1JHumb6oTdkeTqxcdXJ/nHjlmO29qmX7gyM74OVVVJbkmyp7V20xGfGuIaLMs/0jWYKX04D0Psw2VG2oe6kCV04TwMsQ+XGWkf6kKW0IXzMcReXGaUvagLWWLoLky2VR8OsReXGWkv6sNNztNan7vCquryJH+aZEeSW1trf9glyAmoqrfl8HQ2SXYm+asR8lfVXye5NMmbkuxP8qkkX0ryd0l+JMm/J/ml1tos3+RnSf5Lc/i2qJbkySQfW3ttu7mpqp9M8vUkDyc5tDj8yRx+TbvZX4Nj5L8qg1yDudKHW0sX9qULWUYXbi1d2JcuZBlduPX0YT+6kGVG7sJkzD7UhX3pw03O02vgAQAAAAAAsFl6vaQVAAAAAADApjHwAAAAAAAAhmfgAQAAAAAADM/AAwAAAAAAGJ6BBwAAAAAAMDwDDwAAAAAAYHgGHgAAAAAAwPD+D/nutN7AgPYwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 2016x2016 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(28,28))\n",
    "for i in range(5):  \n",
    "    plt.subplot(1, 5, i+1)\n",
    "    plt.title(y_train[i], fontsize=100)\n",
    "    plt.imshow(x_train[i])\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({4: 5267,\n",
       "         2: 5356,\n",
       "         5: 4875,\n",
       "         8: 5275,\n",
       "         1: 6046,\n",
       "         3: 5495,\n",
       "         6: 5291,\n",
       "         9: 5388,\n",
       "         7: 5641,\n",
       "         0: 5366})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter_train = Counter(y_train)\n",
    "counter_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({7: 1028,\n",
       "         2: 1032,\n",
       "         1: 1135,\n",
       "         0: 980,\n",
       "         4: 982,\n",
       "         9: 1009,\n",
       "         5: 892,\n",
       "         6: 958,\n",
       "         3: 1010,\n",
       "         8: 974})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter_test = Counter(y_test)\n",
    "counter_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 28, 1)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_shape = (x_train.shape[1], x_train.shape[2], 1)\n",
    "input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-1, 28, 28, 1)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_shape = (-1,) + input_shape\n",
    "x_train_shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CNN의 입력으로 사용하기 위해서는 Input data의 shape가 (batch, witdh, height, channel)이어야 합니다.  \n",
    "이번 실습에서는 (60000, 28, 28, 1)이 적당합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.reshape(x_train_shape)\n",
    "x_val = x_val.reshape(x_train_shape)\n",
    "x_test = x_test.reshape(x_train_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54000, 28, 28, 1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input Normalization\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_val = x_val.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# 주의!! 이 박스가 두번 이상 실행되지 않도록 주의할 것."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07450981],\n",
       "        [0.19607843],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.6       ],\n",
       "        [0.96862745],\n",
       "        [0.22352941],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05490196],\n",
       "        [0.78039217],\n",
       "        [0.7647059 ],\n",
       "        [0.1882353 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.27450982],\n",
       "        [0.5686275 ],\n",
       "        [0.99215686],\n",
       "        [0.6       ],\n",
       "        [0.29803923],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.23137255],\n",
       "        [0.99215686],\n",
       "        [0.7254902 ],\n",
       "        [0.13333334],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.27450982],\n",
       "        [0.05490196],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.09019608],\n",
       "        [1.        ],\n",
       "        [0.99607843],\n",
       "        [0.30588236],\n",
       "        [0.0627451 ],\n",
       "        [0.09019608],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.7137255 ],\n",
       "        [0.5803922 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01568628],\n",
       "        [0.7254902 ],\n",
       "        [0.99607843],\n",
       "        [0.7490196 ],\n",
       "        [0.04705882],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.21568628],\n",
       "        [0.9411765 ],\n",
       "        [0.99215686],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.27450982],\n",
       "        [0.99215686],\n",
       "        [0.99607843],\n",
       "        [0.26666668],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.13333334],\n",
       "        [0.96862745],\n",
       "        [0.99215686],\n",
       "        [0.78431374],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.28627452],\n",
       "        [0.95686275],\n",
       "        [0.99215686],\n",
       "        [0.37254903],\n",
       "        [0.01176471],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.3254902 ],\n",
       "        [0.87058824],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.3137255 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.0627451 ],\n",
       "        [0.8352941 ],\n",
       "        [0.99215686],\n",
       "        [0.69411767],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [1.        ],\n",
       "        [0.99607843],\n",
       "        [0.99607843],\n",
       "        [0.37254903],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01960784],\n",
       "        [0.23137255],\n",
       "        [0.23137255],\n",
       "        [0.5254902 ],\n",
       "        [0.6117647 ],\n",
       "        [0.7882353 ],\n",
       "        [0.99607843],\n",
       "        [0.99607843],\n",
       "        [0.69803923],\n",
       "        [0.09019608],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.99607843],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.8509804 ],\n",
       "        [0.8392157 ],\n",
       "        [0.84313726],\n",
       "        [0.8509804 ],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99607843],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.98039216],\n",
       "        [0.9137255 ],\n",
       "        [0.20784314],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.60784316],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99607843],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99215686],\n",
       "        [0.99607843],\n",
       "        [0.99215686],\n",
       "        [0.89411765],\n",
       "        [0.43529412],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01960784],\n",
       "        [0.4       ],\n",
       "        [0.45882353],\n",
       "        [0.45882353],\n",
       "        [0.45882353],\n",
       "        [0.45882353],\n",
       "        [0.13333334],\n",
       "        [0.07450981],\n",
       "        [0.07450981],\n",
       "        [0.6313726 ],\n",
       "        [0.99607843],\n",
       "        [0.91764706],\n",
       "        [0.28627452],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.6156863 ],\n",
       "        [0.99607843],\n",
       "        [0.827451  ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.24313726],\n",
       "        [0.99607843],\n",
       "        [0.85490197],\n",
       "        [0.11764706],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.7137255 ],\n",
       "        [0.99215686],\n",
       "        [0.5019608 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.12156863],\n",
       "        [0.94509804],\n",
       "        [0.99215686],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.23921569],\n",
       "        [0.83137256],\n",
       "        [0.2509804 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]],\n",
       "\n",
       "       [[0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label to one-hot encoding\n",
    "y_train = to_categorical(y_train, num_classes = 10)\n",
    "y_val = to_categorical(y_val, num_classes = 10)\n",
    "y_test = to_categorical(y_test, num_classes = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 1., 0., 0., 0., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 28, 28, 32)        832       \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 28, 28, 32)        25632     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 14, 14, 64)        18496     \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 14, 14, 64)        36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 7, 7, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 7, 7, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 3136)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               803072    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                2570      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 887,530\n",
      "Trainable params: 887,530\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu', input_shape = input_shape))\n",
    "model.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2,2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu'))\n",
    "model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(256, activation = \"relu\"))\n",
    "# model.add(Conv2D(filters = 256, kernel_size = (7,7),padding = 'Same', activation ='relu'))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "# model.add(Dense(10, activation = \"softmax\"))\n",
    "model.add(Dense(10))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "# Define the optimizer\n",
    "optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "# Compile the model\n",
    "model.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a learning rate annealer\n",
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n",
    "                                            patience=3, \n",
    "                                            verbose=1, \n",
    "                                            factor=0.5, \n",
    "                                            min_lr=0.00001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model training with fit_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make data generator\n",
    "datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        zoom_range = 0.1, # Randomly zoom image \n",
    "        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
    "        horizontal_flip=False,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "\n",
    "datagen.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/5\n",
      " - 15s - loss: 0.2573 - acc: 0.9177 - val_loss: 0.0618 - val_acc: 0.9808\n",
      "Epoch 2/5\n",
      " - 12s - loss: 0.0857 - acc: 0.9732 - val_loss: 0.0553 - val_acc: 0.9832\n",
      "Epoch 3/5\n",
      " - 12s - loss: 0.0717 - acc: 0.9789 - val_loss: 0.0389 - val_acc: 0.9897\n",
      "Epoch 4/5\n",
      " - 12s - loss: 0.0605 - acc: 0.9821 - val_loss: 0.0311 - val_acc: 0.9908\n",
      "Epoch 5/5\n",
      " - 12s - loss: 0.0580 - acc: 0.9830 - val_loss: 0.0358 - val_acc: 0.9892\n"
     ]
    }
   ],
   "source": [
    "# Fit the model with model.fit_generator\n",
    "epochs = 5 \n",
    "batch_size = 64\n",
    "\n",
    "history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),\n",
    "                              epochs = epochs, validation_data = (x_val,y_val),\n",
    "                              verbose = 2, steps_per_epoch=x_train.shape[0] // batch_size\n",
    "                              , callbacks=[learning_rate_reduction])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 85us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9937"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(x_test, y_test)\n",
    "test_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_json = model.to_json()\n",
    "with open(\"checkpoints/cnn_model.json\", \"w\") as json_file : \n",
    "    json.dump(model_json, json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights(\"checkpoints/cnn_model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}