shakesphere-py.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WBd69MDEm4rF"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pD_55cOxLkAb"
   },
   "outputs": [],
   "source": [
    "path_to_file = 'shakespeare.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aavnuByVymwK"
   },
   "outputs": [],
   "source": [
    "text = open(path_to_file, 'r').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Duhg9NrUymwO"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                     1\n",
      "  From fairest creatures we desire increase,\n",
      "  That thereby beauty's rose might never die,\n",
      "  But as the riper should by time decease,\n",
      "  His tender heir might bear his memory:\n",
      "  But thou contracted to thine own bright eyes,\n",
      "  Feed'st thy light's flame with self-substantial fuel,\n",
      "  Making a famine where abundance lies,\n",
      "  Thy self thy foe, to thy sweet self too cruel:\n",
      "  Thou that art now the world's fresh ornament,\n",
      "  And only herald to the gaudy spring,\n",
      "  Within thine own bu\n"
     ]
    }
   ],
   "source": [
    "print(text[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IlCgQBRVymwR"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n', ' ', '!', '\"', '&', \"'\", '(', ')', ',', '-', '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '>', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', ']', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '|', '}']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "84"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The unique characters in the file\n",
    "vocab = sorted(set(text))\n",
    "print(vocab)\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IalZLbvOzf-F"
   },
   "outputs": [],
   "source": [
    "char_to_ind = {u:i for i, u in enumerate(vocab)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fmmP5iCwm4rp"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'\\n': 0,\n",
       " ' ': 1,\n",
       " '!': 2,\n",
       " '\"': 3,\n",
       " '&': 4,\n",
       " \"'\": 5,\n",
       " '(': 6,\n",
       " ')': 7,\n",
       " ',': 8,\n",
       " '-': 9,\n",
       " '.': 10,\n",
       " '0': 11,\n",
       " '1': 12,\n",
       " '2': 13,\n",
       " '3': 14,\n",
       " '4': 15,\n",
       " '5': 16,\n",
       " '6': 17,\n",
       " '7': 18,\n",
       " '8': 19,\n",
       " '9': 20,\n",
       " ':': 21,\n",
       " ';': 22,\n",
       " '<': 23,\n",
       " '>': 24,\n",
       " '?': 25,\n",
       " 'A': 26,\n",
       " 'B': 27,\n",
       " 'C': 28,\n",
       " 'D': 29,\n",
       " 'E': 30,\n",
       " 'F': 31,\n",
       " 'G': 32,\n",
       " 'H': 33,\n",
       " 'I': 34,\n",
       " 'J': 35,\n",
       " 'K': 36,\n",
       " 'L': 37,\n",
       " 'M': 38,\n",
       " 'N': 39,\n",
       " 'O': 40,\n",
       " 'P': 41,\n",
       " 'Q': 42,\n",
       " 'R': 43,\n",
       " 'S': 44,\n",
       " 'T': 45,\n",
       " 'U': 46,\n",
       " 'V': 47,\n",
       " 'W': 48,\n",
       " 'X': 49,\n",
       " 'Y': 50,\n",
       " 'Z': 51,\n",
       " '[': 52,\n",
       " ']': 53,\n",
       " '_': 54,\n",
       " '`': 55,\n",
       " 'a': 56,\n",
       " 'b': 57,\n",
       " 'c': 58,\n",
       " 'd': 59,\n",
       " 'e': 60,\n",
       " 'f': 61,\n",
       " 'g': 62,\n",
       " 'h': 63,\n",
       " 'i': 64,\n",
       " 'j': 65,\n",
       " 'k': 66,\n",
       " 'l': 67,\n",
       " 'm': 68,\n",
       " 'n': 69,\n",
       " 'o': 70,\n",
       " 'p': 71,\n",
       " 'q': 72,\n",
       " 'r': 73,\n",
       " 's': 74,\n",
       " 't': 75,\n",
       " 'u': 76,\n",
       " 'v': 77,\n",
       " 'w': 78,\n",
       " 'x': 79,\n",
       " 'y': 80,\n",
       " 'z': 81,\n",
       " '|': 82,\n",
       " '}': 83}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_to_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "30ZYaWAOm4rt"
   },
   "outputs": [],
   "source": [
    "ind_to_char = np.array(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_6JPOWwJm4rz"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['\\n', ' ', '!', '\"', '&', \"'\", '(', ')', ',', '-', '.', '0', '1',\n",
       "       '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '>', '?',\n",
       "       'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',\n",
       "       'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',\n",
       "       '[', ']', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',\n",
       "       'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',\n",
       "       'w', 'x', 'y', 'z', '|', '}'], dtype='<U1')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ind_to_char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3fhOqV0lm4r2"
   },
   "outputs": [],
   "source": [
    "encoded_text = np.array([char_to_ind[c] for c in text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "axOX7rFom4r5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  1, ..., 30, 39, 29])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tFs1Uza-m4r9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n                   '"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample = text[:20]\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gIqUCK5Am4sB"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_text[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pAvUYFk7m4sF"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                     1\n",
      "  From fairest creatures we desire increase,\n",
      "  That thereby beauty's rose might never die,\n",
      "  But as the riper should by time decease,\n",
      "  His tender heir might bear his memory:\n",
      "  But thou contracted to thine own bright eyes,\n",
      "  Feed'st thy light's flame with self-substantial fuel,\n",
      "  Making a famine where abundance lies,\n",
      "  Thy self thy foe, to thy sweet self too cruel:\n",
      "  Thou that art now the world's fresh ornament,\n",
      "  And only herald to the gaudy spring,\n",
      "  Within thine own bu\n"
     ]
    }
   ],
   "source": [
    "print(text[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D45OYgOfm4sJ"
   },
   "outputs": [],
   "source": [
    "line = \"From fairest creatures we desire increase\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7dKiEVN8m4sL"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "olX67f6-m4sP"
   },
   "outputs": [],
   "source": [
    "part_stanza = \"\"\"From fairest creatures we desire increase,\n",
    "  That thereby beauty's rose might never die,\n",
    "  But as the riper should by time decease,\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qal7MQnqm4sQ"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "131"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(part_stanza)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0UHJDA39zf-O"
   },
   "outputs": [],
   "source": [
    "seq_len = 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7VRSK4cOm4sZ"
   },
   "outputs": [],
   "source": [
    "total_num_seq = len(text)//(seq_len+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xtW0jbbvm4sc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45005"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_num_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ciatnowvm4se"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "1\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "F\n",
      "r\n",
      "o\n",
      "m\n",
      " \n",
      "f\n",
      "a\n",
      "i\n",
      "r\n",
      "e\n",
      "s\n",
      "t\n",
      " \n",
      "c\n",
      "r\n",
      "e\n",
      "a\n",
      "t\n",
      "u\n",
      "r\n",
      "e\n",
      "s\n",
      " \n",
      "w\n",
      "e\n",
      " \n",
      "d\n",
      "e\n",
      "s\n",
      "i\n",
      "r\n",
      "e\n",
      " \n",
      "i\n",
      "n\n",
      "c\n",
      "r\n",
      "e\n",
      "a\n",
      "s\n",
      "e\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "T\n",
      "h\n",
      "a\n",
      "t\n",
      " \n",
      "t\n",
      "h\n",
      "e\n",
      "r\n",
      "e\n",
      "b\n",
      "y\n",
      " \n",
      "b\n",
      "e\n",
      "a\n",
      "u\n",
      "t\n",
      "y\n",
      "'\n",
      "s\n",
      " \n",
      "r\n",
      "o\n",
      "s\n",
      "e\n",
      " \n",
      "m\n",
      "i\n",
      "g\n",
      "h\n",
      "t\n",
      " \n",
      "n\n",
      "e\n",
      "v\n",
      "e\n",
      "r\n",
      " \n",
      "d\n",
      "i\n",
      "e\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "B\n",
      "u\n",
      "t\n",
      " \n",
      "a\n",
      "s\n",
      " \n",
      "t\n",
      "h\n",
      "e\n",
      " \n",
      "r\n",
      "i\n",
      "p\n",
      "e\n",
      "r\n",
      " \n",
      "s\n",
      "h\n",
      "o\n",
      "u\n",
      "l\n",
      "d\n",
      " \n",
      "b\n",
      "y\n",
      " \n",
      "t\n",
      "i\n",
      "m\n",
      "e\n",
      " \n",
      "d\n",
      "e\n",
      "c\n",
      "e\n",
      "a\n",
      "s\n",
      "e\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "H\n",
      "i\n",
      "s\n",
      " \n",
      "t\n",
      "e\n",
      "n\n",
      "d\n",
      "e\n",
      "r\n",
      " \n",
      "h\n",
      "e\n",
      "i\n",
      "r\n",
      " \n",
      "m\n",
      "i\n",
      "g\n",
      "h\n",
      "t\n",
      " \n",
      "b\n",
      "e\n",
      "a\n",
      "r\n",
      " \n",
      "h\n",
      "i\n",
      "s\n",
      " \n",
      "m\n",
      "e\n",
      "m\n",
      "o\n",
      "r\n",
      "y\n",
      ":\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "B\n",
      "u\n",
      "t\n",
      " \n",
      "t\n",
      "h\n",
      "o\n",
      "u\n",
      " \n",
      "c\n",
      "o\n",
      "n\n",
      "t\n",
      "r\n",
      "a\n",
      "c\n",
      "t\n",
      "e\n",
      "d\n",
      " \n",
      "t\n",
      "o\n",
      " \n",
      "t\n",
      "h\n",
      "i\n",
      "n\n",
      "e\n",
      " \n",
      "o\n",
      "w\n",
      "n\n",
      " \n",
      "b\n",
      "r\n",
      "i\n",
      "g\n",
      "h\n",
      "t\n",
      " \n",
      "e\n",
      "y\n",
      "e\n",
      "s\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "F\n",
      "e\n",
      "e\n",
      "d\n",
      "'\n",
      "s\n",
      "t\n",
      " \n",
      "t\n",
      "h\n",
      "y\n",
      " \n",
      "l\n",
      "i\n",
      "g\n",
      "h\n",
      "t\n",
      "'\n",
      "s\n",
      " \n",
      "f\n",
      "l\n",
      "a\n",
      "m\n",
      "e\n",
      " \n",
      "w\n",
      "i\n",
      "t\n",
      "h\n",
      " \n",
      "s\n",
      "e\n",
      "l\n",
      "f\n",
      "-\n",
      "s\n",
      "u\n",
      "b\n",
      "s\n",
      "t\n",
      "a\n",
      "n\n",
      "t\n",
      "i\n",
      "a\n",
      "l\n",
      " \n",
      "f\n",
      "u\n",
      "e\n",
      "l\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "M\n",
      "a\n",
      "k\n",
      "i\n",
      "n\n",
      "g\n",
      " \n",
      "a\n",
      " \n",
      "f\n",
      "a\n",
      "m\n",
      "i\n",
      "n\n",
      "e\n",
      " \n",
      "w\n",
      "h\n",
      "e\n",
      "r\n",
      "e\n",
      " \n",
      "a\n",
      "b\n",
      "u\n",
      "n\n",
      "d\n",
      "a\n",
      "n\n",
      "c\n",
      "e\n",
      " \n",
      "l\n",
      "i\n",
      "e\n",
      "s\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "T\n",
      "h\n",
      "y\n",
      " \n",
      "s\n",
      "e\n",
      "l\n",
      "f\n",
      " \n",
      "t\n",
      "h\n",
      "y\n",
      " \n",
      "f\n",
      "o\n",
      "e\n",
      ",\n",
      " \n",
      "t\n",
      "o\n",
      " \n",
      "t\n",
      "h\n",
      "y\n",
      " \n",
      "s\n",
      "w\n",
      "e\n",
      "e\n",
      "t\n",
      " \n",
      "s\n",
      "e\n",
      "l\n",
      "f\n",
      " \n",
      "t\n",
      "o\n",
      "o\n",
      " \n",
      "c\n",
      "r\n",
      "u\n",
      "e\n",
      "l\n",
      ":\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "T\n",
      "h\n",
      "o\n",
      "u\n",
      " \n",
      "t\n",
      "h\n",
      "a\n",
      "t\n",
      " \n",
      "a\n",
      "r\n",
      "t\n",
      " \n",
      "n\n",
      "o\n",
      "w\n",
      " \n",
      "t\n",
      "h\n",
      "e\n",
      " \n",
      "w\n",
      "o\n",
      "r\n",
      "l\n",
      "d\n",
      "'\n",
      "s\n",
      " \n",
      "f\n",
      "r\n",
      "e\n",
      "s\n",
      "h\n",
      " \n",
      "o\n",
      "r\n",
      "n\n",
      "a\n",
      "m\n",
      "e\n",
      "n\n",
      "t\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "A\n",
      "n\n",
      "d\n",
      " \n",
      "o\n",
      "n\n",
      "l\n",
      "y\n",
      " \n",
      "h\n",
      "e\n",
      "r\n",
      "a\n",
      "l\n",
      "d\n",
      " \n",
      "t\n",
      "o\n",
      " \n",
      "t\n",
      "h\n",
      "e\n",
      " \n",
      "g\n",
      "a\n",
      "u\n",
      "d\n",
      "y\n",
      " \n",
      "s\n",
      "p\n",
      "r\n",
      "i\n",
      "n\n",
      "g\n",
      ",\n",
      "\n",
      "\n",
      " \n",
      " \n",
      "W\n",
      "i\n",
      "t\n",
      "h\n",
      "i\n",
      "n\n",
      " \n",
      "t\n",
      "h\n",
      "i\n",
      "n\n",
      "e\n",
      " \n",
      "o\n",
      "w\n",
      "n\n",
      " \n",
      "b\n",
      "u\n"
     ]
    }
   ],
   "source": [
    "# Create Training Sequences\n",
    "char_dataset = tf.data.Dataset.from_tensor_slices(encoded_text)\n",
    "\n",
    "for i in char_dataset.take(500):\n",
    "     print(ind_to_char[i.numpy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l4hkDU3i7ozi"
   },
   "outputs": [],
   "source": [
    "sequences = char_dataset.batch(seq_len+1, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9NGu-FkO_kYU"
   },
   "outputs": [],
   "source": [
    "def create_seq_targets(seq):\n",
    "    input_txt = seq[:-1]\n",
    "    target_txt = seq[1:]\n",
    "    return input_txt, target_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HszljTg8m4so"
   },
   "outputs": [],
   "source": [
    "dataset = sequences.map(create_seq_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JkPa7AMrm4sq"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1 12  0\n",
      "  1  1 31 73 70 68  1 61 56 64 73 60 74 75  1 58 73 60 56 75 76 73 60 74\n",
      "  1 78 60  1 59 60 74 64 73 60  1 64 69 58 73 60 56 74 60  8  0  1  1 45\n",
      " 63 56 75  1 75 63 60 73 60 57 80  1 57 60 56 76 75 80  5 74  1 73 70 74\n",
      " 60  1 68 64 62 63 75  1 69 60 77 60 73  1 59 64 60  8  0  1  1 27 76 75]\n",
      "\n",
      "                     1\n",
      "  From fairest creatures we desire increase,\n",
      "  That thereby beauty's rose might never die,\n",
      "  But\n",
      "\n",
      "\n",
      "[ 1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1 12  0  1\n",
      "  1 31 73 70 68  1 61 56 64 73 60 74 75  1 58 73 60 56 75 76 73 60 74  1\n",
      " 78 60  1 59 60 74 64 73 60  1 64 69 58 73 60 56 74 60  8  0  1  1 45 63\n",
      " 56 75  1 75 63 60 73 60 57 80  1 57 60 56 76 75 80  5 74  1 73 70 74 60\n",
      "  1 68 64 62 63 75  1 69 60 77 60 73  1 59 64 60  8  0  1  1 27 76 75  1]\n",
      "                     1\n",
      "  From fairest creatures we desire increase,\n",
      "  That thereby beauty's rose might never die,\n",
      "  But \n"
     ]
    }
   ],
   "source": [
    "for input_txt, target_txt in  dataset.take(1):\n",
    "    print(input_txt.numpy())\n",
    "    print(''.join(ind_to_char[input_txt.numpy()]))\n",
    "    print('\\n')\n",
    "    print(target_txt.numpy())\n",
    "    # There is an extra whitespace!\n",
    "    print(''.join(ind_to_char[target_txt.numpy()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p2pGotuNzf-S"
   },
   "outputs": [],
   "source": [
    "# Batch size\n",
    "batch_size = 128\n",
    "\n",
    "# Buffer size to shuffle the dataset so it doesn't attempt to shuffle\n",
    "# the entire sequence in memory. Instead, it maintains a buffer in which it shuffles elements\n",
    "buffer_size = 10000\n",
    "\n",
    "dataset = dataset.shuffle(buffer_size).batch(batch_size, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gmcCALymm4su"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<BatchDataset shapes: ((128, 120), (128, 120)), types: (tf.int32, tf.int32)>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zHT8cLh7EAsg"
   },
   "outputs": [],
   "source": [
    "# Length of the vocabulary in chars\n",
    "vocab_size = len(vocab)\n",
    "\n",
    "# The embedding dimension\n",
    "embed_dim = 64\n",
    "\n",
    "# Number of RNN units\n",
    "rnn_neurons = 1026"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YeRlEXgym4s1"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM,Dense,Embedding,Dropout,GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VoFVGKlNkJfW"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.losses import sparse_categorical_crossentropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sblCzZoslZKH"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function sparse_categorical_crossentropy in module tensorflow.python.keras.losses:\n",
      "\n",
      "sparse_categorical_crossentropy(y_true, y_pred, from_logits=False, axis=-1)\n",
      "    Computes the sparse categorical crossentropy loss.\n",
      "    \n",
      "    Usage:\n",
      "    \n",
      "    >>> y_true = [1, 2]\n",
      "    >>> y_pred = [[0.05, 0.95, 0], [0.1, 0.8, 0.1]]\n",
      "    >>> loss = tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred)\n",
      "    >>> assert loss.shape == (2,)\n",
      "    >>> loss.numpy()\n",
      "    array([0.0513, 2.303], dtype=float32)\n",
      "    \n",
      "    Args:\n",
      "      y_true: Ground truth values.\n",
      "      y_pred: The predicted values.\n",
      "      from_logits: Whether `y_pred` is expected to be a logits tensor. By default,\n",
      "        we assume that `y_pred` encodes a probability distribution.\n",
      "      axis: (Optional) Defaults to -1. The dimension along which the entropy is\n",
      "        computed.\n",
      "    \n",
      "    Returns:\n",
      "      Sparse categorical crossentropy loss value.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(sparse_categorical_crossentropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FrOOK61Olm1C"
   },
   "outputs": [],
   "source": [
    "def sparse_cat_loss(y_true,y_pred):\n",
    "  return sparse_categorical_crossentropy(y_true, y_pred, from_logits=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MtCrdfzEI2N0"
   },
   "outputs": [],
   "source": [
    "def create_model(vocab_size, embed_dim, rnn_neurons, batch_size):\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(vocab_size, embed_dim,batch_input_shape=[batch_size, None]))\n",
    "    model.add(GRU(rnn_neurons,return_sequences=True,stateful=True,recurrent_initializer='glorot_uniform'))\n",
    "    # Final Dense Layer to Predict\n",
    "    model.add(Dense(vocab_size))\n",
    "    model.compile(optimizer='adam', loss=sparse_cat_loss) \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wwsrpOik5zhv"
   },
   "outputs": [],
   "source": [
    "model = create_model(\n",
    "  vocab_size = vocab_size,\n",
    "  embed_dim=embed_dim,\n",
    "  rnn_neurons=rnn_neurons,\n",
    "  batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "liXuTFYMm4s6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (128, None, 64)           5376      \n",
      "_________________________________________________________________\n",
      "gru (GRU)                    (128, None, 1026)         3361176   \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (128, None, 84)           86268     \n",
      "=================================================================\n",
      "Total params: 3,452,820\n",
      "Trainable params: 3,452,820\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "A4ygvfHn-wan"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(128, 120, 84)  <=== (batch_size, sequence_length, vocab_size)\n"
     ]
    }
   ],
   "source": [
    "for input_example_batch, target_example_batch in dataset.take(1):\n",
    "\n",
    "  # Predict off some random batch\n",
    "  example_batch_predictions = model(input_example_batch)\n",
    "\n",
    "  # Display the dimensions of the predictions\n",
    "  print(example_batch_predictions.shape, \" <=== (batch_size, sequence_length, vocab_size)\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5ld8z3LPBAuv"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(128, 120, 84), dtype=float32, numpy=\n",
       "array([[[ 6.8499269e-03,  5.1499852e-03,  4.1904575e-03, ...,\n",
       "          1.1563038e-03,  8.6244312e-04,  1.7285278e-03],\n",
       "        [-2.2295883e-03,  7.6984540e-03,  4.7736213e-04, ...,\n",
       "         -5.4260152e-03, -4.1469499e-03, -4.2291936e-03],\n",
       "        [ 9.8236278e-04, -4.6168902e-04, -1.6062905e-03, ...,\n",
       "         -4.3694684e-03, -6.0256077e-03, -5.5640824e-03],\n",
       "        ...,\n",
       "        [-8.2467211e-04,  2.8963230e-04, -2.8553749e-03, ...,\n",
       "         -6.8982854e-03,  3.4304531e-03,  5.1585231e-03],\n",
       "        [-1.3885333e-03, -1.3720469e-03,  5.5692242e-03, ...,\n",
       "         -3.4013255e-03, -5.6886941e-04,  5.7104777e-04],\n",
       "        [-4.2776647e-03,  2.5034323e-03,  2.4291605e-03, ...,\n",
       "         -6.0919528e-03, -2.7628483e-03, -4.7401134e-03]],\n",
       "\n",
       "       [[ 7.1117835e-04,  1.9175004e-03,  4.3472350e-03, ...,\n",
       "         -1.2502901e-02, -4.9306643e-03, -6.7475284e-03],\n",
       "        [-4.4840197e-03,  5.1361411e-03,  2.5019359e-03, ...,\n",
       "         -7.2485260e-03, -5.7251672e-03, -5.8696261e-03],\n",
       "        [ 5.0030220e-03,  3.8053836e-03,  2.0485082e-04, ...,\n",
       "         -8.0149807e-03, -1.8992590e-03, -1.5270023e-03],\n",
       "        ...,\n",
       "        [-1.3856835e-03,  7.7020732e-04,  7.4004256e-03, ...,\n",
       "          1.3407787e-03, -2.1279969e-03, -3.6713919e-03],\n",
       "        [ 7.3177246e-03,  5.7168370e-03,  7.0512695e-03, ...,\n",
       "          8.3576643e-04,  2.5311214e-04, -2.0810037e-03],\n",
       "        [ 8.6908685e-03,  1.0504177e-02, -7.7871222e-04, ...,\n",
       "         -7.1003735e-03,  4.3604928e-03, -2.4772744e-06]],\n",
       "\n",
       "       [[ 6.8499269e-03,  5.1499852e-03,  4.1904575e-03, ...,\n",
       "          1.1563038e-03,  8.6244312e-04,  1.7285278e-03],\n",
       "        [ 6.5403717e-04,  6.4131063e-03, -6.3400571e-03, ...,\n",
       "         -6.3822470e-03, -2.4907303e-03, -1.4244746e-03],\n",
       "        [ 2.6184113e-03,  5.3670024e-04, -2.7224631e-03, ...,\n",
       "         -7.8407936e-03,  3.9161323e-03,  3.3784299e-03],\n",
       "        ...,\n",
       "        [-4.2017805e-03, -1.7935730e-03,  1.2042180e-02, ...,\n",
       "         -3.6174124e-03, -1.7783809e-03, -6.7150313e-04],\n",
       "        [-3.2120552e-03, -1.4796904e-03,  1.3930622e-02, ...,\n",
       "         -1.1517016e-04, -3.1493758e-03, -5.4459102e-05],\n",
       "        [ 3.8283337e-03,  6.1151399e-03,  3.3876668e-03, ...,\n",
       "         -7.7684405e-03,  3.0092353e-03,  6.0437853e-04]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[-4.7660089e-04,  3.0405694e-03, -1.3886763e-03, ...,\n",
       "          7.2995666e-04, -1.0055098e-03,  4.1733342e-03],\n",
       "        [-1.0173374e-02,  1.4861283e-03, -3.0078450e-03, ...,\n",
       "          8.8576172e-03, -1.0115753e-03,  1.4675825e-03],\n",
       "        [-5.3471858e-03, -3.3164076e-03, -5.2562119e-03, ...,\n",
       "          3.4210528e-03, -1.6662371e-03, -1.2970441e-03],\n",
       "        ...,\n",
       "        [ 6.7409495e-04,  1.3695338e-03,  7.8264019e-03, ...,\n",
       "         -1.2875230e-02, -5.9623104e-03, -1.0624253e-02],\n",
       "        [ 6.9226651e-03,  6.2437928e-03,  7.2380062e-03, ...,\n",
       "         -4.7534672e-03, -2.5411416e-03, -4.2654732e-03],\n",
       "        [-3.7129459e-03, -4.4418359e-03,  6.0295979e-03, ...,\n",
       "         -2.0344865e-03, -3.2672303e-04, -5.0146459e-03]],\n",
       "\n",
       "       [[ 4.9080416e-03,  7.2527314e-03, -3.1219604e-03, ...,\n",
       "         -6.3997931e-03,  4.0135640e-03,  1.2712143e-03],\n",
       "        [-3.6589184e-03,  4.1380376e-03, -3.9094533e-03, ...,\n",
       "         -1.9482645e-03, -6.2939734e-03,  1.6502835e-03],\n",
       "        [-3.5848236e-03,  7.5032108e-04,  5.0501027e-03, ...,\n",
       "         -1.5506303e-03, -4.9276934e-03, -1.0248476e-03],\n",
       "        ...,\n",
       "        [ 1.5408744e-04,  1.6188616e-03,  6.4711152e-03, ...,\n",
       "         -9.2535466e-04, -5.6737191e-03, -2.0183721e-03],\n",
       "        [ 6.5669841e-03, -5.0616087e-03,  2.4577226e-03, ...,\n",
       "          3.6594993e-03,  2.3616303e-03, -6.2261889e-04],\n",
       "        [-1.0977418e-04,  1.1552812e-03,  1.3639973e-03, ...,\n",
       "         -1.9229572e-03, -9.2411024e-04, -4.5861583e-03]],\n",
       "\n",
       "       [[-1.1153095e-02, -5.5710576e-04, -2.9771309e-03, ...,\n",
       "          7.4733472e-03,  5.0965646e-05, -2.4660211e-04],\n",
       "        [ 2.3526272e-04, -4.6042628e-03, -2.0317279e-03, ...,\n",
       "          8.7742237e-03,  5.6869653e-03,  1.5455651e-03],\n",
       "        [ 2.0021389e-03, -2.6852081e-03,  9.6879713e-04, ...,\n",
       "          1.9840542e-03,  2.3515793e-03,  2.2531960e-04],\n",
       "        ...,\n",
       "        [-2.3064176e-03,  3.7874508e-04,  8.2010927e-04, ...,\n",
       "         -7.3149297e-03, -1.3336383e-03, -8.5808672e-03],\n",
       "        [-1.1129312e-03, -2.6117200e-03,  6.7903581e-03, ...,\n",
       "         -4.1561644e-03, -2.9215193e-03, -6.5737544e-03],\n",
       "        [-4.5369738e-03, -5.6109461e-03,  9.6877962e-03, ...,\n",
       "         -6.5364931e-03, -3.0328017e-03, -4.3591466e-03]]], dtype=float32)>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_batch_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_achqjT-BGyY"
   },
   "outputs": [],
   "source": [
    "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xWrPFk2nBJX4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(120, 1), dtype=int64, numpy=\n",
       "array([[22],\n",
       "       [69],\n",
       "       [32],\n",
       "       [10],\n",
       "       [22],\n",
       "       [51],\n",
       "       [29],\n",
       "       [64],\n",
       "       [77],\n",
       "       [28],\n",
       "       [70],\n",
       "       [26],\n",
       "       [20],\n",
       "       [56],\n",
       "       [ 5],\n",
       "       [50],\n",
       "       [77],\n",
       "       [69],\n",
       "       [20],\n",
       "       [24],\n",
       "       [66],\n",
       "       [76],\n",
       "       [46],\n",
       "       [59],\n",
       "       [54],\n",
       "       [12],\n",
       "       [54],\n",
       "       [19],\n",
       "       [42],\n",
       "       [ 2],\n",
       "       [81],\n",
       "       [27],\n",
       "       [82],\n",
       "       [11],\n",
       "       [55],\n",
       "       [ 8],\n",
       "       [62],\n",
       "       [48],\n",
       "       [76],\n",
       "       [36],\n",
       "       [67],\n",
       "       [79],\n",
       "       [21],\n",
       "       [16],\n",
       "       [59],\n",
       "       [55],\n",
       "       [22],\n",
       "       [38],\n",
       "       [75],\n",
       "       [ 2],\n",
       "       [15],\n",
       "       [68],\n",
       "       [19],\n",
       "       [72],\n",
       "       [65],\n",
       "       [52],\n",
       "       [39],\n",
       "       [67],\n",
       "       [24],\n",
       "       [40],\n",
       "       [72],\n",
       "       [ 9],\n",
       "       [57],\n",
       "       [39],\n",
       "       [44],\n",
       "       [43],\n",
       "       [ 2],\n",
       "       [63],\n",
       "       [60],\n",
       "       [66],\n",
       "       [ 8],\n",
       "       [51],\n",
       "       [43],\n",
       "       [59],\n",
       "       [77],\n",
       "       [63],\n",
       "       [56],\n",
       "       [34],\n",
       "       [ 8],\n",
       "       [12],\n",
       "       [51],\n",
       "       [12],\n",
       "       [59],\n",
       "       [13],\n",
       "       [ 7],\n",
       "       [38],\n",
       "       [72],\n",
       "       [81],\n",
       "       [61],\n",
       "       [45],\n",
       "       [10],\n",
       "       [43],\n",
       "       [29],\n",
       "       [22],\n",
       "       [55],\n",
       "       [14],\n",
       "       [64],\n",
       "       [63],\n",
       "       [61],\n",
       "       [67],\n",
       "       [77],\n",
       "       [ 6],\n",
       "       [22],\n",
       "       [58],\n",
       "       [ 5],\n",
       "       [21],\n",
       "       [70],\n",
       "       [82],\n",
       "       [ 5],\n",
       "       [72],\n",
       "       [14],\n",
       "       [43],\n",
       "       [54],\n",
       "       [75],\n",
       "       [14],\n",
       "       [68],\n",
       "       [29],\n",
       "       [15],\n",
       "       [20],\n",
       "       [36]], dtype=int64)>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Wi80PQVtBLqj"
   },
   "outputs": [],
   "source": [
    "# Reformat to not be a lists of lists\n",
    "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4qYkIg00-wjq"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([22, 69, 32, 10, 22, 51, 29, 64, 77, 28, 70, 26, 20, 56,  5, 50, 77,\n",
       "       69, 20, 24, 66, 76, 46, 59, 54, 12, 54, 19, 42,  2, 81, 27, 82, 11,\n",
       "       55,  8, 62, 48, 76, 36, 67, 79, 21, 16, 59, 55, 22, 38, 75,  2, 15,\n",
       "       68, 19, 72, 65, 52, 39, 67, 24, 40, 72,  9, 57, 39, 44, 43,  2, 63,\n",
       "       60, 66,  8, 51, 43, 59, 77, 63, 56, 34,  8, 12, 51, 12, 59, 13,  7,\n",
       "       38, 72, 81, 61, 45, 10, 43, 29, 22, 55, 14, 64, 63, 61, 67, 77,  6,\n",
       "       22, 58,  5, 21, 70, 82,  5, 72, 14, 43, 54, 75, 14, 68, 29, 15, 20,\n",
       "       36], dtype=int64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H9-P_XqQ_7wY"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Given the input seq: \n",
      "\n",
      " to me they are strong. The Queen his mother\n",
      "    Lives almost by his looks; and for myself,-\n",
      "    My virtue or my plague,\n",
      "\n",
      "\n",
      "Next Char Predictions: \n",
      "\n",
      ";nG.;ZDivCoA9a'Yvn9>kuUd_1_8Q!zB|0`,gWuKlx:5d`;Mt!4m8qj[Nl>Oq-bNSR!hek,ZRdvhaI,1Z1d2)MqzfT.RD;`3ihflv(;c':o|'q3R_t3mD49K\n"
     ]
    }
   ],
   "source": [
    "print(\"Given the input seq: \\n\")\n",
    "print(\"\".join(ind_to_char[input_example_batch[0]]))\n",
    "print('\\n')\n",
    "print(\"Next Char Predictions: \\n\")\n",
    "print(\"\".join(ind_to_char[sampled_indices ]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZYDQjKTlm4s8"
   },
   "outputs": [],
   "source": [
    "epochs = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_PJ4OVdBm4s8"
   },
   "outputs": [],
   "source": [
    "model.fit(dataset,epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eYRNG57Govdc"
   },
   "outputs": [],
   "source": [
    "model.save('shakespeare_gen.h5') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GCoJayFS8H4d"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_iXG3VJvEXWM"
   },
   "outputs": [],
   "source": [
    "model = create_model(vocab_size, embed_dim, rnn_neurons, batch_size=1)\n",
    "\n",
    "model.load_weights('shakespeare_gen.h5')\n",
    "\n",
    "model.build(tf.TensorShape([1, None]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LAX3p7_YEilU"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (1, None, 64)             5376      \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (1, None, 1026)           3361176   \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (1, None, 84)             86268     \n",
      "=================================================================\n",
      "Total params: 3,452,820\n",
      "Trainable params: 3,452,820\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WvuwZBX5Ogfd"
   },
   "outputs": [],
   "source": [
    "def generate_text(model, start_seed,gen_size=100,temp=1.0):\n",
    "  '''\n",
    "  model: Trained Model to Generate Text\n",
    "  start_seed: Intial Seed text in string form\n",
    "  gen_size: Number of characters to generate\n",
    "\n",
    "  Basic idea behind this function is to take in some seed text, format it so\n",
    "  that it is in the correct shape for our network, then loop the sequence as\n",
    "  we keep adding our own predicted characters. Similar to our work in the RNN\n",
    "  time series problems.\n",
    "  '''\n",
    "\n",
    "  # Number of characters to generate\n",
    "  num_generate = gen_size\n",
    "\n",
    "  # Vecotrizing starting seed text\n",
    "  input_eval = [char_to_ind[s] for s in start_seed]\n",
    "\n",
    "  # Expand to match batch format shape\n",
    "  input_eval = tf.expand_dims(input_eval, 0)\n",
    "\n",
    "  # Empty list to hold resulting generated text\n",
    "  text_generated = []\n",
    "\n",
    " \n",
    "  temperature = temp\n",
    "\n",
    "  # Here batch size == 1\n",
    "  model.reset_states()\n",
    "\n",
    "  for i in range(num_generate):\n",
    "\n",
    "      # Generate Predictions\n",
    "      predictions = model(input_eval)\n",
    "\n",
    "      # Remove the batch shape dimension\n",
    "      predictions = tf.squeeze(predictions, 0)\n",
    "\n",
    "      # Use a cateogircal disitribution to select the next character\n",
    "      predictions = predictions / temperature\n",
    "      predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n",
    "\n",
    "      # Pass the predicted charracter for the next input\n",
    "      input_eval = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "      # Transform back to character letter\n",
    "      text_generated.append(ind_to_char[predicted_id])\n",
    "\n",
    "  return (start_seed + ''.join(text_generated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bS69SG5D5lwd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flower, Casca, am I offended by yet\n",
      "    Our ladyes; Paud, for 'tis agreed's head.\n",
      "  DUKE. Marry, sir, I can congreet you trifling doom\n",
      "    And laid this am wine fruitfully;\n",
      "    All those fishes, arms to a kind of sleep.\n",
      "  JULIA. Caesar!                       They fight.\n",
      "  MARCUS. And therefore, let us go in respect of each\n",
      "    outrage, and as if she denounce my father cryst\n",
      "    A thousand Music.  \n",
      "  BANNILAN. O Mercury, good my lords?\n",
      "  ANTIGONUS.  [Aside]  I'll warrant you, I pray God old Douglas know.\n",
      "  JESSICA. Nay, I will plag and cry too long.                    Exeunt\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SCENE III.\n",
      "The peop of him, KING EDWARD and VIOLA\n",
      "\n",
      "  ELLO. I would bring me to these heads a leamness of mine,\n",
      "    Relieve hot his helmers to your habit one Lucure\n",
      "    of VALENTRADES were crestful\n",
      "    Of what she will assay it but that indeed\n",
      "    No honest employments to this town.\n",
      "    Here is married and her usurping grain.\n",
      "  PUBETEUS. Why, thou wilt first be, pardon mighty wit, thy wife's crown is, ill\n",
      "    unconstan\n"
     ]
    }
   ],
   "source": [
    "print(generate_text(model,\"flower\",gen_size=1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dff5xCHIeEHO"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "00-Generating-Text-with-RNNs.ipynb",
   "private_outputs": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}