03.02.01.text_classification_Conv1D.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification with 1-d CNN\n",
    "\n",
    "이전 실습 중에 LSTM을 이용하여 Text를 state vector로 encoding하고, 이를 Feature 로 활용하여 Text classification하는 IMDB dataset 기반 sentimental analysis task를 다루어 본 바 있습니다.  \n",
    "\n",
    "본 실습에서는 동일한 task를 1-d CNN Encoder를 활용하여 다시 다루어 보겠습니다. 이를 통해 Feature를 추출하는 관점에 따른 모델의 특성 및 분류성능 차이를 확인해 보겠습니다.\n",
    "\n",
    "(참고)  \n",
    "https://github.com/gilbutITbook/006975/blob/master/6.4-sequence-processing-with-convnets.ipynb  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from scipy.spatial.distance import cdist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tf.keras.models import Sequential  # This does not work!\n",
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Dense, Conv1D, MaxPooling1D, GlobalMaxPooling1D, Embedding\n",
    "from tensorflow.python.keras.optimizers import Adam\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train-set size:  25000\n",
      "Test-set size:   25000\n"
     ]
    }
   ],
   "source": [
    "# import imdb\n",
    "from keras.datasets import imdb\n",
    "\n",
    "# 데이터 관련 설정은 LSTM 케이스와 동일하게 한다.\n",
    "max_features = 10000\n",
    "max_tokens = 580\n",
    "embedding_size = 8\n",
    "\n",
    "# save np.load\n",
    "np_load_old = np.load\n",
    "# modify the default parameters of np.load\n",
    "np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k)\n",
    "# call load_data with allow_pickle implicitly set to true\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)   # 원래는 이 라인만 있으면 된다.\n",
    "# restore np.load for future normal usage\n",
    "np.load = np_load_old\n",
    "\n",
    "print(\"Train-set size: \", len(x_train))\n",
    "print(\"Test-set size:  \", len(x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 580)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pad = 'pre'\n",
    "x_train_pad = pad_sequences(x_train, maxlen=max_tokens, padding=pad, truncating=pad)\n",
    "x_test_pad = pad_sequences(x_test, maxlen=max_tokens, padding=pad, truncating=pad)\n",
    "x_train_pad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    1,  194, 1153,  194, 8255,\n",
       "         78,  228,    5,    6, 1463, 4369, 5012,  134,   26,    4,  715,\n",
       "          8,  118, 1634,   14,  394,   20,   13,  119,  954,  189,  102,\n",
       "          5,  207,  110, 3103,   21,   14,   69,  188,    8,   30,   23,\n",
       "          7,    4,  249,  126,   93,    4,  114,    9, 2300, 1523,    5,\n",
       "        647,    4,  116,    9,   35, 8163,    4,  229,    9,  340, 1322,\n",
       "          4,  118,    9,    4,  130, 4901,   19,    4, 1002,    5,   89,\n",
       "         29,  952,   46,   37,    4,  455,    9,   45,   43,   38, 1543,\n",
       "       1905,  398,    4, 1649,   26, 6853,    5,  163,   11, 3215,    2,\n",
       "          4, 1153,    9,  194,  775,    7, 8255,    2,  349, 2637,  148,\n",
       "        605,    2, 8003,   15,  123,  125,   68,    2, 6853,   15,  349,\n",
       "        165, 4362,   98,    5,    4,  228,    9,   43,    2, 1157,   15,\n",
       "        299,  120,    5,  120,  174,   11,  220,  175,  136,   50,    9,\n",
       "       4373,  228, 8255,    5,    2,  656,  245, 2350,    5,    4, 9837,\n",
       "        131,  152,  491,   18,    2,   32, 7464, 1212,   14,    9,    6,\n",
       "        371,   78,   22,  625,   64, 1382,    9,    8,  168,  145,   23,\n",
       "          4, 1690,   15,   16,    4, 1355,    5,   28,    6,   52,  154,\n",
       "        462,   33,   89,   78,  285,   16,  145,   95])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_pad[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터셋 구성은 LSTM 모델의 경우와 동일합니다.  \n",
    "\n",
    "## Create Model with Conv1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "layer_embedding (Embedding)  (None, 580, 8)            80000     \n",
      "_________________________________________________________________\n",
      "conv1d_11 (Conv1D)           (None, 574, 32)           1824      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1 (None, 114, 32)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 108, 32)           7200      \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_4 (Glob (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 89,057\n",
      "Trainable params: 89,057\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=max_features,\n",
    "                    output_dim=embedding_size,\n",
    "                    input_length=max_tokens,\n",
    "                    name='layer_embedding'))\n",
    "model.add(Conv1D(32, 7, activation='relu'))\n",
    "model.add(MaxPooling1D(5))\n",
    "model.add(Conv1D(32, 7, activation='relu'))\n",
    "model.add(GlobalMaxPooling1D())\n",
    "model.add(Dense(1))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(optimizer=Adam(lr=1e-5),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'layer_embedding_1/embedding_lookup/Identity_1:0' shape=(?, 580, 8) dtype=float32>,\n",
       " <tf.Tensor 'conv1d_11/Relu:0' shape=(?, 574, 32) dtype=float32>,\n",
       " <tf.Tensor 'max_pooling1d_5/Squeeze:0' shape=(?, 114, 32) dtype=float32>,\n",
       " <tf.Tensor 'conv1d_12/Relu:0' shape=(?, 108, 32) dtype=float32>,\n",
       " <tf.Tensor 'global_max_pooling1d_4/Max:0' shape=(?, 32) dtype=float32>,\n",
       " <tf.Tensor 'dense_4/BiasAdd:0' shape=(?, 1) dtype=float32>]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_outputs = [layer.output for layer in model.layers]\n",
    "layer_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 23750 samples, validate on 1250 samples\n",
      "Epoch 1/100\n",
      "23750/23750 [==============================] - 3s 107us/sample - loss: 1.4042 - acc: 0.5000 - val_loss: 1.1238 - val_acc: 0.5008\n",
      "Epoch 2/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 1.0121 - acc: 0.5000 - val_loss: 0.9251 - val_acc: 0.5008\n",
      "Epoch 3/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.8702 - acc: 0.5000 - val_loss: 0.8232 - val_acc: 0.5008\n",
      "Epoch 4/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.7900 - acc: 0.5000 - val_loss: 0.7611 - val_acc: 0.5008\n",
      "Epoch 5/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.7405 - acc: 0.5000 - val_loss: 0.7238 - val_acc: 0.5008\n",
      "Epoch 6/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.7120 - acc: 0.5000 - val_loss: 0.7040 - val_acc: 0.5008\n",
      "Epoch 7/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6982 - acc: 0.5008 - val_loss: 0.6958 - val_acc: 0.5048\n",
      "Epoch 8/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.6930 - acc: 0.5197 - val_loss: 0.6931 - val_acc: 0.5240\n",
      "Epoch 9/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.6910 - acc: 0.5423 - val_loss: 0.6920 - val_acc: 0.5288\n",
      "Epoch 10/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6898 - acc: 0.5486 - val_loss: 0.6912 - val_acc: 0.5384\n",
      "Epoch 11/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6888 - acc: 0.5565 - val_loss: 0.6904 - val_acc: 0.5440\n",
      "Epoch 12/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.6877 - acc: 0.5657 - val_loss: 0.6895 - val_acc: 0.5456\n",
      "Epoch 13/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.6865 - acc: 0.5763 - val_loss: 0.6886 - val_acc: 0.5504\n",
      "Epoch 14/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6853 - acc: 0.5811 - val_loss: 0.6876 - val_acc: 0.5512\n",
      "Epoch 15/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.6840 - acc: 0.5913 - val_loss: 0.6866 - val_acc: 0.5616\n",
      "Epoch 16/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6827 - acc: 0.6005 - val_loss: 0.6854 - val_acc: 0.5752\n",
      "Epoch 17/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.6813 - acc: 0.6120 - val_loss: 0.6842 - val_acc: 0.5824\n",
      "Epoch 18/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.6798 - acc: 0.6216 - val_loss: 0.6830 - val_acc: 0.5792\n",
      "Epoch 19/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6781 - acc: 0.6242 - val_loss: 0.6815 - val_acc: 0.5984\n",
      "Epoch 20/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.6764 - acc: 0.6377 - val_loss: 0.6799 - val_acc: 0.6032\n",
      "Epoch 21/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.6744 - acc: 0.6482 - val_loss: 0.6780 - val_acc: 0.6056\n",
      "Epoch 22/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6720 - acc: 0.6512 - val_loss: 0.6756 - val_acc: 0.6464\n",
      "Epoch 23/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.6691 - acc: 0.6733 - val_loss: 0.6729 - val_acc: 0.6472\n",
      "Epoch 24/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.6659 - acc: 0.6844 - val_loss: 0.6700 - val_acc: 0.6576\n",
      "Epoch 25/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.6624 - acc: 0.6985 - val_loss: 0.6667 - val_acc: 0.6680\n",
      "Epoch 26/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6585 - acc: 0.7013 - val_loss: 0.6630 - val_acc: 0.6808\n",
      "Epoch 27/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.6542 - acc: 0.7109 - val_loss: 0.6590 - val_acc: 0.6952\n",
      "Epoch 28/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.6494 - acc: 0.7228 - val_loss: 0.6541 - val_acc: 0.6968\n",
      "Epoch 29/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.6438 - acc: 0.7271 - val_loss: 0.6485 - val_acc: 0.6968\n",
      "Epoch 30/100\n",
      "23750/23750 [==============================] - 2s 101us/sample - loss: 0.6375 - acc: 0.7381 - val_loss: 0.6424 - val_acc: 0.6928\n",
      "Epoch 31/100\n",
      "23750/23750 [==============================] - 2s 96us/sample - loss: 0.6304 - acc: 0.7355 - val_loss: 0.6353 - val_acc: 0.7080\n",
      "Epoch 32/100\n",
      "23750/23750 [==============================] - 2s 97us/sample - loss: 0.6225 - acc: 0.7470 - val_loss: 0.6276 - val_acc: 0.7056\n",
      "Epoch 33/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6137 - acc: 0.7517 - val_loss: 0.6191 - val_acc: 0.7112\n",
      "Epoch 34/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.6040 - acc: 0.7537 - val_loss: 0.6098 - val_acc: 0.7208\n",
      "Epoch 35/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.5936 - acc: 0.7577 - val_loss: 0.6001 - val_acc: 0.7264\n",
      "Epoch 36/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.5833 - acc: 0.7623 - val_loss: 0.5902 - val_acc: 0.7304\n",
      "Epoch 37/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.5721 - acc: 0.7673 - val_loss: 0.5797 - val_acc: 0.7336\n",
      "Epoch 38/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.5601 - acc: 0.7724 - val_loss: 0.5682 - val_acc: 0.7416\n",
      "Epoch 39/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.5475 - acc: 0.7770 - val_loss: 0.5563 - val_acc: 0.7456\n",
      "Epoch 40/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.5339 - acc: 0.7809 - val_loss: 0.5444 - val_acc: 0.7496\n",
      "Epoch 41/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.5208 - acc: 0.7869 - val_loss: 0.5323 - val_acc: 0.7560\n",
      "Epoch 42/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.5081 - acc: 0.7919 - val_loss: 0.5206 - val_acc: 0.7672\n",
      "Epoch 43/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.4941 - acc: 0.7979 - val_loss: 0.5097 - val_acc: 0.7752\n",
      "Epoch 44/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.4805 - acc: 0.8023 - val_loss: 0.5086 - val_acc: 0.7776\n",
      "Epoch 45/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.4688 - acc: 0.8084 - val_loss: 0.4994 - val_acc: 0.7832\n",
      "Epoch 46/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.4579 - acc: 0.8123 - val_loss: 0.4911 - val_acc: 0.7856\n",
      "Epoch 47/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.4497 - acc: 0.8155 - val_loss: 0.4867 - val_acc: 0.7928\n",
      "Epoch 48/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.4421 - acc: 0.8197 - val_loss: 0.4803 - val_acc: 0.7912\n",
      "Epoch 49/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.4365 - acc: 0.8240 - val_loss: 0.4768 - val_acc: 0.7968\n",
      "Epoch 50/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.4310 - acc: 0.8273 - val_loss: 0.4729 - val_acc: 0.7992\n",
      "Epoch 51/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.4254 - acc: 0.8313 - val_loss: 0.4688 - val_acc: 0.8016\n",
      "Epoch 52/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.4198 - acc: 0.8336 - val_loss: 0.4771 - val_acc: 0.8016\n",
      "Epoch 53/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.4143 - acc: 0.8365 - val_loss: 0.4649 - val_acc: 0.8056\n",
      "Epoch 54/100\n",
      "23750/23750 [==============================] - 3s 108us/sample - loss: 0.4085 - acc: 0.8388 - val_loss: 0.4798 - val_acc: 0.8104\n",
      "Epoch 55/100\n",
      "23750/23750 [==============================] - 2s 98us/sample - loss: 0.4030 - acc: 0.8408 - val_loss: 0.4808 - val_acc: 0.8104\n",
      "Epoch 56/100\n",
      "23750/23750 [==============================] - 2s 99us/sample - loss: 0.3967 - acc: 0.8432 - val_loss: 0.4752 - val_acc: 0.8136\n",
      "Epoch 57/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.3910 - acc: 0.8458 - val_loss: 0.4711 - val_acc: 0.8160\n",
      "Epoch 58/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23750/23750 [==============================] - 2s 100us/sample - loss: 0.3852 - acc: 0.8486 - val_loss: 0.4668 - val_acc: 0.8176\n",
      "Epoch 59/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.3795 - acc: 0.8514 - val_loss: 0.4626 - val_acc: 0.8192\n",
      "Epoch 60/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.3733 - acc: 0.8544 - val_loss: 0.4587 - val_acc: 0.8208\n",
      "Epoch 61/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.3667 - acc: 0.8567 - val_loss: 0.4539 - val_acc: 0.8216\n",
      "Epoch 62/100\n",
      "23750/23750 [==============================] - 2s 101us/sample - loss: 0.3601 - acc: 0.8585 - val_loss: 0.4510 - val_acc: 0.8224\n",
      "Epoch 63/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.3547 - acc: 0.8608 - val_loss: 0.4664 - val_acc: 0.8208\n",
      "Epoch 64/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.3480 - acc: 0.8635 - val_loss: 0.4718 - val_acc: 0.8256\n",
      "Epoch 65/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.3421 - acc: 0.8647 - val_loss: 0.4598 - val_acc: 0.8272\n",
      "Epoch 66/100\n",
      "23750/23750 [==============================] - 2s 96us/sample - loss: 0.3371 - acc: 0.8677 - val_loss: 0.4744 - val_acc: 0.8240\n",
      "Epoch 67/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.3321 - acc: 0.8700 - val_loss: 0.4703 - val_acc: 0.8288\n",
      "Epoch 68/100\n",
      "23750/23750 [==============================] - 3s 117us/sample - loss: 0.3268 - acc: 0.8721 - val_loss: 0.4753 - val_acc: 0.8280\n",
      "Epoch 69/100\n",
      "23750/23750 [==============================] - 2s 101us/sample - loss: 0.3215 - acc: 0.8747 - val_loss: 0.4728 - val_acc: 0.8312\n",
      "Epoch 70/100\n",
      "23750/23750 [==============================] - 2s 97us/sample - loss: 0.3167 - acc: 0.8760 - val_loss: 0.4788 - val_acc: 0.8360\n",
      "Epoch 71/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.3141 - acc: 0.8774 - val_loss: 0.4781 - val_acc: 0.8288\n",
      "Epoch 72/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.3088 - acc: 0.8796 - val_loss: 0.4655 - val_acc: 0.8384\n",
      "Epoch 73/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.3052 - acc: 0.8814 - val_loss: 0.4629 - val_acc: 0.8416\n",
      "Epoch 74/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.3018 - acc: 0.8828 - val_loss: 0.4611 - val_acc: 0.8416\n",
      "Epoch 75/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.2982 - acc: 0.8843 - val_loss: 0.4595 - val_acc: 0.8416\n",
      "Epoch 76/100\n",
      "23750/23750 [==============================] - 2s 98us/sample - loss: 0.2946 - acc: 0.8859 - val_loss: 0.4582 - val_acc: 0.8408\n",
      "Epoch 77/100\n",
      "23750/23750 [==============================] - 2s 96us/sample - loss: 0.2909 - acc: 0.8869 - val_loss: 0.4579 - val_acc: 0.8392\n",
      "Epoch 78/100\n",
      "23750/23750 [==============================] - 2s 93us/sample - loss: 0.2874 - acc: 0.8879 - val_loss: 0.4573 - val_acc: 0.8400\n",
      "Epoch 79/100\n",
      "23750/23750 [==============================] - 2s 91us/sample - loss: 0.2838 - acc: 0.8894 - val_loss: 0.4654 - val_acc: 0.8432\n",
      "Epoch 80/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.2804 - acc: 0.8904 - val_loss: 0.4732 - val_acc: 0.8424\n",
      "Epoch 81/100\n",
      "23750/23750 [==============================] - 2s 104us/sample - loss: 0.2769 - acc: 0.8917 - val_loss: 0.4809 - val_acc: 0.8464\n",
      "Epoch 82/100\n",
      "23750/23750 [==============================] - 2s 98us/sample - loss: 0.2735 - acc: 0.8925 - val_loss: 0.4879 - val_acc: 0.8464\n",
      "Epoch 83/100\n",
      "23750/23750 [==============================] - 2s 98us/sample - loss: 0.2701 - acc: 0.8938 - val_loss: 0.4873 - val_acc: 0.8456\n",
      "Epoch 84/100\n",
      "23750/23750 [==============================] - 2s 96us/sample - loss: 0.2668 - acc: 0.8953 - val_loss: 0.4951 - val_acc: 0.8488\n",
      "Epoch 85/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.2636 - acc: 0.8956 - val_loss: 0.5024 - val_acc: 0.8488\n",
      "Epoch 86/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.2606 - acc: 0.8964 - val_loss: 0.5094 - val_acc: 0.8496\n",
      "Epoch 87/100\n",
      "23750/23750 [==============================] - 2s 92us/sample - loss: 0.2577 - acc: 0.8982 - val_loss: 0.5084 - val_acc: 0.8496\n",
      "Epoch 88/100\n",
      "23750/23750 [==============================] - 2s 97us/sample - loss: 0.2547 - acc: 0.8986 - val_loss: 0.5065 - val_acc: 0.8504\n",
      "Epoch 89/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.2518 - acc: 0.9002 - val_loss: 0.5055 - val_acc: 0.8496\n",
      "Epoch 90/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.2497 - acc: 0.9008 - val_loss: 0.4980 - val_acc: 0.8528\n",
      "Epoch 91/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.2475 - acc: 0.9015 - val_loss: 0.5042 - val_acc: 0.8496\n",
      "Epoch 92/100\n",
      "23750/23750 [==============================] - 2s 94us/sample - loss: 0.2444 - acc: 0.9024 - val_loss: 0.5057 - val_acc: 0.8488\n",
      "Epoch 93/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.2424 - acc: 0.9031 - val_loss: 0.5024 - val_acc: 0.8520\n",
      "Epoch 94/100\n",
      "23750/23750 [==============================] - 2s 104us/sample - loss: 0.2403 - acc: 0.9041 - val_loss: 0.5019 - val_acc: 0.8520\n",
      "Epoch 95/100\n",
      "23750/23750 [==============================] - 2s 96us/sample - loss: 0.2381 - acc: 0.9048 - val_loss: 0.5021 - val_acc: 0.8520\n",
      "Epoch 96/100\n",
      "23750/23750 [==============================] - 2s 101us/sample - loss: 0.2359 - acc: 0.9061 - val_loss: 0.5112 - val_acc: 0.8512\n",
      "Epoch 97/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.2337 - acc: 0.9065 - val_loss: 0.5100 - val_acc: 0.8512\n",
      "Epoch 98/100\n",
      "23750/23750 [==============================] - 2s 95us/sample - loss: 0.2313 - acc: 0.9075 - val_loss: 0.5088 - val_acc: 0.8520\n",
      "Epoch 99/100\n",
      "23750/23750 [==============================] - 2s 96us/sample - loss: 0.2291 - acc: 0.9088 - val_loss: 0.5082 - val_acc: 0.8536\n",
      "Epoch 100/100\n",
      "23750/23750 [==============================] - 2s 97us/sample - loss: 0.2268 - acc: 0.9093 - val_loss: 0.5080 - val_acc: 0.8552\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1f74677c940>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train_pad, y_train,\n",
    "          validation_split=0.05, epochs=100, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 [==============================] - 2s 69us/sample - loss: 0.5082 - acc: 0.8506\n"
     ]
    }
   ],
   "source": [
    "result = model.evaluate(x_test_pad, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 85.06%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: {0:.2%}\".format(result[1]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}