GPT_2_Anime.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GPT_2_Anime",
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d467f151813f4a24900389cafa24ce61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4d5470a48f5d42639de9ad0bce3a0049",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0b84d4420b2342b7bb904c3084e1667d",
              "IPY_MODEL_235456569fab454b8d2c290908bca1a5"
            ]
          }
        },
        "4d5470a48f5d42639de9ad0bce3a0049": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "0b84d4420b2342b7bb904c3084e1667d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6cefddbfa2924c85bdd72c7a165e0302",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1248751,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1248751,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_51034ec8d3a04f169c1c31f877392826"
          }
        },
        "235456569fab454b8d2c290908bca1a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f1c2e29c557343838125b740d704f1cd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1248751/1248751 [00:36&lt;00:00, 33825.63it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03d5b1dc90834d53a65d4420ba5fdf72"
          }
        },
        "6cefddbfa2924c85bdd72c7a165e0302": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "51034ec8d3a04f169c1c31f877392826": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1c2e29c557343838125b740d704f1cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03d5b1dc90834d53a65d4420ba5fdf72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d3074436f9814326bc8fa887ff299ee6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bc2fa11b8033436eab8773b48544fb99",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fa0035b9eac64b2a95ce5ceb71402f40",
              "IPY_MODEL_effe3af98e304f48beba0e2d5289a748"
            ]
          }
        },
        "bc2fa11b8033436eab8773b48544fb99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "fa0035b9eac64b2a95ce5ceb71402f40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_34fae027e44540038ef0fd50be9ddfce",
            "_dom_classes": [],
            "description": "Loss: 2.094 — Avg: 2.095 — GPU Mem: 13797 MB: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4972287716ee40d3b715b677cbb9a9d1"
          }
        },
        "effe3af98e304f48beba0e2d5289a748": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5ea10b4e54584a74a20259f5ee1065c1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [7:03:11&lt;00:00,  3.94it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e013d4c7f59f449383f1f9e1b7113f0d"
          }
        },
        "34fae027e44540038ef0fd50be9ddfce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4972287716ee40d3b715b677cbb9a9d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5ea10b4e54584a74a20259f5ee1065c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e013d4c7f59f449383f1f9e1b7113f0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0yrd5h0gCHkz"
      },
      "source": [
        "# GPT-2 Anime Subtitle Generation\n",
        "\n",
        "## tl;dr\n",
        "1. `Connect` or `Reconnect`\n",
        "2. Upload your Kaggle API key (instructions included later)\n",
        "3. `Runtime` -> `Restart and run all`\n",
        "4. Wait 15-30 minutes\n",
        "5. Laugh at weird computer-generated Anime Subtitles\n",
        "\n",
        "\n",
        "by Brian Lechthaler, \n",
        "*based on [aitextgen](https://github.com/minimaxir/aitextgen)*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hOHCCCA6CWwB"
      },
      "source": [
        "# Dependencies\n",
        "Download and install all necessary dependencies with `pip`, then `import` what we need."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5sulzsy5WHC0",
        "outputId": "c8f9f19e-f446-4398-ee8f-97e497c93824",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install -q kaggle\n",
        "# Freeze versions of dependencies for now\n",
        "!pip install -q transformers==2.9.1\n",
        "!pip install -q pytorch-lightning==0.7.6\n",
        "\n",
        "!pip install -q aitextgen\n",
        "\n",
        "import logging\n",
        "logging.basicConfig(\n",
        "        format=\"%(asctime)s — %(levelname)s — %(name)s — %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        level=logging.INFO\n",
        "    )\n",
        "\n",
        "from aitextgen import aitextgen\n",
        "from aitextgen.colab import mount_gdrive, copy_file_from_gdrive\n",
        "from aitextgen.TokenDataset import TokenDataset, merge_datasets\n",
        "from aitextgen.utils import build_gpt2_config\n",
        "from aitextgen.tokenizers import train_tokenizer"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "11/15/2020 00:28:40 — INFO — transformers.file_utils — PyTorch version 1.7.0+cu101 available.\n",
            "11/15/2020 00:28:42 — INFO — transformers.file_utils — TensorFlow version 2.3.0 available.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eII1AiBFCbEV"
      },
      "source": [
        "# Mount Google Drive\n",
        "Because any data in the VM this notebook is running on will be nuked once the Jupyter kernel stops running, it's helpful to mount your Google Drive to the Colab VM to persist some files that we'll use in this notebook.\n",
        "\n",
        "*Note:* your data will not be shared with anyone who does not have direct access to the VM running this Colab notebook."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HYznWbR_WSl3",
        "outputId": "436ff136-7285-4a8d-b84c-adbf9cb5d19d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "mount_gdrive()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GBoUdBX-CfiG"
      },
      "source": [
        "# Download Dataset from Kaggle\n",
        "Downloads the 'kickstarter-projects' dataset contributed by Kaggle user `jef1056`\n",
        "\n",
        "1.   Sign into Kaggle in a separate tab\n",
        "2.   Click [this link](https://kaggle.com/me/account) to go to your Kaggle account settings\n",
        "3. Under the `API` section, click/tap `Create new API token`. If this is not the first time you have followed this step, consider clicking `Expire API Token` prior to generating a new token.\n",
        "4. In the Colab file browser, upload the `kaggle.json` API token you just downloaded in step 3.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c_TLQ9yZPPga",
        "outputId": "e57a625a-c560-45b3-89f4-aea11e42b8ed",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!mkdir -p /root/.kaggle\n",
        "!mv kaggle.json /root/.kaggle/kaggle.json\n",
        "!chmod 600 /root/.kaggle/kaggle.json\n",
        "!rm -rf anime-subtitles.zip\n",
        "!kaggle datasets download -d jef1056/anime-subtitles\n",
        "!rm -rf 'Anime Datasets V3.zip'\n",
        "!rm -rf 'input (Cleaned).txt'\n",
        "!unzip anime-subtitles.zip\n",
        "!wc -l 'input (Cleaned).txt'"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading anime-subtitles.zip to /content\n",
            " 92% 129M/140M [00:01<00:00, 50.6MB/s] \n",
            "100% 140M/140M [00:02<00:00, 70.9MB/s]\n",
            "Archive:  anime-subtitles.zip\n",
            "  inflating: Anime Datasets V3.zip   \n",
            "  inflating: input (Cleaned).txt     \n",
            "1248751 input (Cleaned).txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D_cF-EYGCmrd"
      },
      "source": [
        "# Train Tokenizer on Dataset\n",
        "Bound to CPU, may take a few minutes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v1KxWMyNXQsZ"
      },
      "source": [
        "file_name = 'input (Cleaned).txt'\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TfCKKkOsayuX"
      },
      "source": [
        "def cleandir(rm_model):\n",
        "  print('cleaning working directory...')\n",
        "  !rm -rf aitextgen-merges.txt\n",
        "  !rm -rf aitextgen-vocab.json\n",
        "  if rm_model == True:\n",
        "    !rm -rf /content/trained_model\n",
        "  elif rm_model == False:\n",
        "    print('note: rm_model set to False, skipping model deletion.')\n",
        "  else:\n",
        "    print('note: rm_model not set to True or False, skipping model deletion.')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qi3OGNRkn8qP"
      },
      "source": [
        "cleandir(False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iwgXjOmxXcLm",
        "outputId": "23552708-7bec-47f4-961c-5060d99fc216",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "train_tokenizer(file_name)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "11/15/2020 00:29:21 — INFO — aitextgen.tokenizers — Saving aitextgen-vocab.json and aitextgen-merges.txt to the current directory. You will need both files to build the GPT2Tokenizer.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "thbsjNCtCx3S"
      },
      "source": [
        "# Configure GPT-2 Training\n",
        "Set various configuration variables to control how the GPT-2 model is re-trained to the data we are feeding it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gI-RgLNzX2eZ",
        "outputId": "817ede36-77e0-4257-f053-674de8b3ad51",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "config = build_gpt2_config(vocab_size=30000, \n",
        "                           max_length=64, \n",
        "                           dropout=0.0, \n",
        "                           n_embd=256, \n",
        "                           n_layer=8, \n",
        "                           n_head=8)\n",
        "config"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2Config {\n",
              "  \"activation_function\": \"gelu_new\",\n",
              "  \"attn_pdrop\": 0.0,\n",
              "  \"bos_token_id\": 0,\n",
              "  \"embd_pdrop\": 0.0,\n",
              "  \"eos_token_id\": 0,\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"layer_norm_epsilon\": 1e-05,\n",
              "  \"model_type\": \"gpt2\",\n",
              "  \"n_ctx\": 64,\n",
              "  \"n_embd\": 256,\n",
              "  \"n_head\": 8,\n",
              "  \"n_layer\": 8,\n",
              "  \"n_positions\": 64,\n",
              "  \"resid_pdrop\": 0.0,\n",
              "  \"summary_activation\": null,\n",
              "  \"summary_first_dropout\": 0.0,\n",
              "  \"summary_proj_to_labels\": true,\n",
              "  \"summary_type\": \"cls_index\",\n",
              "  \"summary_use_proj\": true,\n",
              "  \"vocab_size\": 30000\n",
              "}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c91npByBX5is",
        "outputId": "c49ae372-6e8e-4222-bac8-26501db552ce",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "ai = aitextgen(config=config,\n",
        "               vocab_file=\"aitextgen-vocab.json\",\n",
        "               merges_file=\"aitextgen-merges.txt\",\n",
        "               to_gpu=True)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "11/15/2020 00:29:21 — INFO — aitextgen — Constructing GPT-2 model from provided config.\n",
            "11/15/2020 00:29:22 — INFO — aitextgen — Using a custom tokenizer.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G3nFDdkiDBnt"
      },
      "source": [
        "# Re-train GPT-2 to Dataset\n",
        "\n",
        "This task is bound to the GPU and should take just under two hours to train on an NVidia V100 GPU.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nTf7z7GEasfo"
      },
      "source": [
        "!rm -rf trained_model"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hWIbNQuHYBK5",
        "outputId": "c584c87f-20f3-4dfe-d1a6-3071d56c47af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "d467f151813f4a24900389cafa24ce61",
            "4d5470a48f5d42639de9ad0bce3a0049",
            "0b84d4420b2342b7bb904c3084e1667d",
            "235456569fab454b8d2c290908bca1a5",
            "6cefddbfa2924c85bdd72c7a165e0302",
            "51034ec8d3a04f169c1c31f877392826",
            "f1c2e29c557343838125b740d704f1cd",
            "03d5b1dc90834d53a65d4420ba5fdf72",
            "d3074436f9814326bc8fa887ff299ee6",
            "bc2fa11b8033436eab8773b48544fb99",
            "fa0035b9eac64b2a95ce5ceb71402f40",
            "effe3af98e304f48beba0e2d5289a748",
            "34fae027e44540038ef0fd50be9ddfce",
            "4972287716ee40d3b715b677cbb9a9d1",
            "5ea10b4e54584a74a20259f5ee1065c1",
            "e013d4c7f59f449383f1f9e1b7113f0d"
          ]
        }
      },
      "source": [
        "ai.train(file_name,\n",
        "         line_by_line=True,\n",
        "         num_steps=100000,\n",
        "         generate_every=1000,\n",
        "         save_every=500,\n",
        "         save_gdrive=False,\n",
        "         learning_rate=1e-4,\n",
        "         batch_size=256)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d467f151813f4a24900389cafa24ce61",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=1248751.0), HTML(value='')), layout=Layou…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "11/15/2020 00:29:36 — INFO — aitextgen.TokenDataset — Encoding 1,248,751 sets of tokens from input (Cleaned).txt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "GPU available: True, used: True\n",
            "11/15/2020 00:30:14 — INFO — lightning — GPU available: True, used: True\n",
            "No environment variable for node rank defined. Set as 0.\n",
            "11/15/2020 00:30:14 — WARNING — lightning — No environment variable for node rank defined. Set as 0.\n",
            "CUDA_VISIBLE_DEVICES: [0]\n",
            "11/15/2020 00:30:14 — INFO — lightning — CUDA_VISIBLE_DEVICES: [0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d3074436f9814326bc8fa887ff299ee6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=100000.0), HTML(value='')), layout=Layout…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[1m500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m1,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m1,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">And that it makes the pula is the  way I'll take her to that.\n",
            "\n",
            "==========\n",
            "\u001b[1m1,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m2,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m2,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I just want to use a little like this!\n",
            "\n",
            "==========\n",
            "\u001b[1m2,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m3,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m3,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">My turn!\n",
            "\n",
            "==========\n",
            "\u001b[1m3,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m4,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m4,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The truth is, you're just a good person.\n",
            "\n",
            "==========\n",
            "\u001b[1m4,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m5,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m5,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Suzu, I'm still here.\n",
            "\n",
            "==========\n",
            "\u001b[1m5,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m6,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m6,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Huh?\n",
            "\n",
            "==========\n",
            "\u001b[1m6,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m7,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m7,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I won't let you get away with this!\n",
            "\n",
            "==========\n",
            "\u001b[1m7,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m8,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m8,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'll be waiting for this, you two.\n",
            "\n",
            "==========\n",
            "\u001b[1m8,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m9,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m9,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">It's time for you to start moving.\n",
            "\n",
            "==========\n",
            "\u001b[1m9,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m10,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m10,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">and do you have the same love.\n",
            "\n",
            "==========\n",
            "\u001b[1m10,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m11,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m11,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I don't think...\n",
            "\n",
            "==========\n",
            "\u001b[1m11,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m12,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m12,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">But it's too heavy!\n",
            "\n",
            "==========\n",
            "\u001b[1m12,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m13,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m13,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">We are just kids, but I don't know.\n",
            "\n",
            "==========\n",
            "\u001b[1m13,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m14,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m14,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I want to get the rest to those guys.\n",
            "\n",
            "==========\n",
            "\u001b[1m14,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m15,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m15,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">That's right.\n",
            "\n",
            "==========\n",
            "\u001b[1m15,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m16,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m16,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You're not going to make it up to me.\n",
            "\n",
            "==========\n",
            "\u001b[1m16,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m17,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m17,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">That's a lie.\n",
            "\n",
            "==========\n",
            "\u001b[1m17,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m18,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m18,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I know you're here.\n",
            "\n",
            "==========\n",
            "\u001b[1m18,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m19,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m19,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">She was on the way home.\n",
            "\n",
            "==========\n",
            "\u001b[1m19,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m20,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m20,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Okay, fine.\n",
            "\n",
            "==========\n",
            "\u001b[1m20,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m21,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m21,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Well, I've never seen you two before.\n",
            "\n",
            "==========\n",
            "\u001b[1m21,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m22,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m22,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Is that the power of the Millennium Items?\n",
            "\n",
            "==========\n",
            "\u001b[1m22,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m23,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m23,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">He was so strong...\n",
            "\n",
            "==========\n",
            "\u001b[1m23,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m24,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m24,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I just wanted to talk to you.\n",
            "\n",
            "==========\n",
            "\u001b[1m24,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m25,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m25,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">They're just like that, too.\n",
            "\n",
            "==========\n",
            "\u001b[1m25,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m26,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m26,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Respect!\n",
            "\n",
            "==========\n",
            "\u001b[1m26,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m27,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m27,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'll keep the hell out of this girl!\n",
            "\n",
            "==========\n",
            "\u001b[1m27,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m28,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m28,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The first letter that the other day.\n",
            "\n",
            "==========\n",
            "\u001b[1m28,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m29,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m29,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">the world needed him most, he vanished.\n",
            "\n",
            "==========\n",
            "\u001b[1m29,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m30,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m30,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I got it, Mr. Ren.\n",
            "\n",
            "==========\n",
            "\u001b[1m30,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m31,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m31,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You know, I don't know where I am being dressed...\n",
            "\n",
            "==========\n",
            "\u001b[1m31,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m32,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m32,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">No matter what you say, you're not getting any closer to it.\n",
            "\n",
            "==========\n",
            "\u001b[1m32,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m33,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m33,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Kunikida-kun said she'd  be in their next match.\n",
            "\n",
            "==========\n",
            "\u001b[1m33,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m34,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m34,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">No.\n",
            "\n",
            "==========\n",
            "\u001b[1m34,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m35,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m35,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Mister!\n",
            "\n",
            "==========\n",
            "\u001b[1m35,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m36,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m36,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I see...\n",
            "\n",
            "==========\n",
            "\u001b[1m36,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m37,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m37,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">It's a new world, isn't it?\n",
            "\n",
            "==========\n",
            "\u001b[1m37,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m38,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m38,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Don't tell me, it's all my fault?\n",
            "\n",
            "==========\n",
            "\u001b[1m38,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m39,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m39,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I have to say, I'm really glad I can come on, but...\n",
            "\n",
            "==========\n",
            "\u001b[1m39,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m40,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m40,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Tonight is going to the mill to the palace.\n",
            "\n",
            "==========\n",
            "\u001b[1m40,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m41,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m41,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You can't do it.\n",
            "\n",
            "==========\n",
            "\u001b[1m41,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m42,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m42,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">No, she is not lying.\n",
            "\n",
            "==========\n",
            "\u001b[1m42,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m43,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m43,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'm a loser like you, Kana-chan.\n",
            "\n",
            "==========\n",
            "\u001b[1m43,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m44,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m44,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The pillar of the vessel\n",
            "\n",
            "==========\n",
            "\u001b[1m44,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m45,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m45,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">No, it's not!\n",
            "\n",
            "==========\n",
            "\u001b[1m45,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m46,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m46,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I want to be alone for a moment\n",
            "\n",
            "==========\n",
            "\u001b[1m46,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m47,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m47,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">S-S-S-S-S-S-\n",
            "\n",
            "==========\n",
            "\u001b[1m47,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m48,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m48,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Huh?\n",
            "\n",
            "==========\n",
            "\u001b[1m48,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m49,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m49,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">This is...\n",
            "\n",
            "==========\n",
            "\u001b[1m49,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m50,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m50,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">What, are you saying?\n",
            "\n",
            "==========\n",
            "\u001b[1m50,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m51,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m51,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I bet I could make him make a mistake.\n",
            "\n",
            "==========\n",
            "\u001b[1m51,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m52,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m52,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">So I'm not going to go back to your home.\n",
            "\n",
            "==========\n",
            "\u001b[1m52,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m53,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m53,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Avdol!\n",
            "\n",
            "==========\n",
            "\u001b[1m53,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m54,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m54,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">It's a battle of spikes  that have never reached\n",
            "\n",
            "==========\n",
            "\u001b[1m54,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m55,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m55,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">He's on the roof, then?\n",
            "\n",
            "==========\n",
            "\u001b[1m55,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m56,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m56,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The world is full of things I want to protect\n",
            "\n",
            "==========\n",
            "\u001b[1m56,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m57,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m57,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">He's gonna be a bit late.\n",
            "\n",
            "==========\n",
            "\u001b[1m57,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m58,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m58,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">It's a good thing to do, isn't it?\n",
            "\n",
            "==========\n",
            "\u001b[1m58,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m59,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m59,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'm sure you're the one who's the daughter of the family,\n",
            "\n",
            "==========\n",
            "\u001b[1m59,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m60,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m60,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'll tell them all about that.\n",
            "\n",
            "==========\n",
            "\u001b[1m60,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m61,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m61,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'm sure that was a lie...\n",
            "\n",
            "==========\n",
            "\u001b[1m61,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m62,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m62,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I think you should say \"I love you\"\n",
            "\n",
            "==========\n",
            "\u001b[1m62,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m63,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m63,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'll take care of this.\n",
            "\n",
            "==========\n",
            "\u001b[1m63,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m64,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m64,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The enemy is the remaining 50,000 years ago.\n",
            "\n",
            "==========\n",
            "\u001b[1m64,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m65,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m65,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Toyohisa arrival his moment with you.\n",
            "\n",
            "==========\n",
            "\u001b[1m65,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m66,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m66,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">What?!\n",
            "\n",
            "==========\n",
            "\u001b[1m66,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m67,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m67,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I see...\n",
            "\n",
            "==========\n",
            "\u001b[1m67,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m68,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m68,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">All right, everyone.\n",
            "\n",
            "==========\n",
            "\u001b[1m68,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m69,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m69,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">What are you saying?!\n",
            "\n",
            "==========\n",
            "\u001b[1m69,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m70,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m70,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Don't you think we need to meet up with someone you like?\n",
            "\n",
            "==========\n",
            "\u001b[1m70,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m71,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m71,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'm not a fool, but I  think I'll show you my true power!\n",
            "\n",
            "==========\n",
            "\u001b[1m71,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m72,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m72,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I...\n",
            "\n",
            "==========\n",
            "\u001b[1m72,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m73,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m73,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I had no idea what had happened to you in the past.\n",
            "\n",
            "==========\n",
            "\u001b[1m73,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m74,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m74,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Are you going to go to Tottori for me?\n",
            "\n",
            "==========\n",
            "\u001b[1m74,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m75,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m75,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Touka!\n",
            "\n",
            "==========\n",
            "\u001b[1m75,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m76,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m76,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'm sure we can meet up in the scorching,  and the hymgamins are in a hurry.\n",
            "\n",
            "==========\n",
            "\u001b[1m76,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m77,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m77,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You know, for the first time, you've been in the same class as Kazu-san and Nobu-sensei ever since.\n",
            "\n",
            "==========\n",
            "\u001b[1m77,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m78,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m78,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Let's go home.\n",
            "\n",
            "==========\n",
            "\u001b[1m78,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m79,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m79,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Kimidori-san, you did indeed have the power to see the face of a real boy.\n",
            "\n",
            "==========\n",
            "\u001b[1m79,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m80,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m80,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">But I can't let them do as they please.\n",
            "\n",
            "==========\n",
            "\u001b[1m80,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m81,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m81,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You have to be brave and no one else, right?\n",
            "\n",
            "==========\n",
            "\u001b[1m81,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m82,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m82,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I can't stand you.\n",
            "\n",
            "==========\n",
            "\u001b[1m82,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m83,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m83,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Huh?\n",
            "\n",
            "==========\n",
            "\u001b[1m83,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m84,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m84,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Mashiro, what's the matter?\n",
            "\n",
            "==========\n",
            "\u001b[1m84,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m85,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m85,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I don't know what to say to her.\n",
            "\n",
            "==========\n",
            "\u001b[1m85,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m86,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m86,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The girl who's in the bathroom cafarily died.\n",
            "\n",
            "==========\n",
            "\u001b[1m86,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m87,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m87,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You're welcome.\n",
            "\n",
            "==========\n",
            "\u001b[1m87,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m88,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m88,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I have a favor for you.\n",
            "\n",
            "==========\n",
            "\u001b[1m88,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m89,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m89,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Now, then...\n",
            "\n",
            "==========\n",
            "\u001b[1m89,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m90,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m90,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">No, you're wrong.\n",
            "\n",
            "==========\n",
            "\u001b[1m90,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m91,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m91,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I can't forgive myself for being scared of losing my portion.\n",
            "\n",
            "==========\n",
            "\u001b[1m91,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m92,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m92,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I wonder if you will like it?\n",
            "\n",
            "==========\n",
            "\u001b[1m92,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m93,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m93,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">It's my fault!\n",
            "\n",
            "==========\n",
            "\u001b[1m93,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m94,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m94,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The first scent of the dragon in the village.\n",
            "\n",
            "==========\n",
            "\u001b[1m94,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m95,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m95,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">This is no time to be fated to fight!\n",
            "\n",
            "==========\n",
            "\u001b[1m95,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m96,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m96,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">I'm not scared anymore!\n",
            "\n",
            "==========\n",
            "\u001b[1m96,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m97,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m97,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">Is that you, Ange?\n",
            "\n",
            "==========\n",
            "\u001b[1m97,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m98,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m98,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">You're...\n",
            "\n",
            "==========\n",
            "\u001b[1m98,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m99,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m99,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">In the end, you'd look better  than I thought.\n",
            "\n",
            "==========\n",
            "\u001b[1m99,500 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m100,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m100,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">But, it's your fault.\n",
            "\n",
            "==========\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "11/15/2020 07:33:07 — INFO — aitextgen — Saving trained model pytorch_model.bin to /trained_model\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XOJZs076DSgs"
      },
      "source": [
        "# Generate Samples\n",
        "Finally, the fun part! Have the model generate 25 unique samples. As you can see, the results are quite believable. Certain nuances specific to Japanese to English translation such as (name)-chan (-chan = boy) or (name)-kun (-kun = man) are learned and replicated in generated output, made extra impressive by the fact that everything in this notebook is unsupervised learning meaning *we never told the robot what to do, it figured it out 100% on it's own!*. Even though it's a stretch to say this is applicable to AI-generated Anime subtitles, please use this code responsibly, never to intentionally deceive or do evil with."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vsb7-rtti8Sc",
        "outputId": "2a903d49-3204-4c00-d7b9-9dc4e431f4fa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "ai.generate(n=25,\n",
        "            batch_size=16384,\n",
        "            prompt=\">\",\n",
        "            temperature=1,\n",
        "            top_p=0.99999)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1m>\u001b[0mYou must have heard the rumor that the voices of the weretiger appeared near the street.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mWhat's the matter, Yananza?\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mWell then, I want to eat you.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mYou should be grateful to me.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mThe truth is that none  answer Diana to your existence\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mI had to live up here  just to recover the order.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mIf you try out those  things and just learn how...\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mWe've got to come up with a story about we do!\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mSubaru-sama...\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mThey need to make  sense of how dangerous it is.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mYou okay?\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mHuh?\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mIt's a real problem!\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mThis is quite the unforgettable rained\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mWhat the hell was that, r-rel jerk?!\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mIs it because there's anyone I can  really go to find something?\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mBut it's a good thing that's...\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mand my family allies\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mI-It's all right. Mm.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mThere's nothing wrong with me\\nin this many secrets!\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mYeah. We don't have any to make the first move.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mRi-\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mIt's the same thing the Syndicate.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mYeah, nothing.\n",
            "\n",
            "==========\n",
            "\u001b[1m>\u001b[0mMy father's the one who's been waiting for all these years.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6IFj1jGzXpQV",
        "outputId": "6e4d3906-dc18-4b37-d5ad-70f7a6d56720",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!export \"model_archive=anime_subtitlegen_$(date +%e_%b_%Y_%H_%M_%S)\" ; mkdir $model_archive ; mv aitextgen-* $model_archive ; mv trained_model $model_archive ; tar -cvf $model_archive.tar $model_archive ; mv $model_archive.tar \"drive/My Drive/\" ; echo \"Model successfully backup up to Google Drive. Feel free to factory reset the runtime.\""
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "anime_subtitlegen_15_Nov_2020_07_33_09/\n",
            "anime_subtitlegen_15_Nov_2020_07_33_09/aitextgen-merges.txt\n",
            "anime_subtitlegen_15_Nov_2020_07_33_09/trained_model/\n",
            "anime_subtitlegen_15_Nov_2020_07_33_09/trained_model/pytorch_model.bin\n",
            "anime_subtitlegen_15_Nov_2020_07_33_09/trained_model/config.json\n",
            "anime_subtitlegen_15_Nov_2020_07_33_09/aitextgen-vocab.json\n",
            "Model successfully backup up to Google Drive. Feel free to factory reset the runtime.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crxydRt20zvP"
      },
      "source": [
        "Last updated:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "obOj-hICIWps",
        "outputId": "2139e4f7-ed46-4a2a-d58f-5053e539a69b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import datetime as dt\n",
        "def \n",
        "print(dt.datetime.now())"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-11-15 07:34:14.823167\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IjGg1f-mBcrd"
      },
      "source": [
        "# Credits\n",
        "\n",
        "This project was made possible by the cumulative efforts of the following parties:\n",
        "\n",
        "Brian Lechthaler *author of this notebook*\n",
        "* https://github.com/brianlechthaler\n",
        "* https://twitter.com/brianlechthaler\n",
        "\n",
        "Max Woolf *author of [aitextgen](https://github.com/minimaxir/aitextgen), the training code this notebook is based on.*\n",
        "* https://minimaxir.com/\n",
        "* https://github.com/minimaxir\n",
        "\n",
        "Jess Fan [author](https://www.kaggle.com/jef1056) of [anime-subtitles](https://www.kaggle.com/jef1056/anime-subtitles) dataset\n",
        "* https://github.com/JEF1056\n",
        "* https://www.linkedin.com/in/jess-fan-677177196/\n",
        "\n",
        "OpenAI *creators of [GPT-2](https://en.wikipedia.org/wiki/OpenAI#GPT-2) model*\n",
        "* https://openai.com \n",
        "* https://openai.com/blog/tags/gpt-2/\n"
      ]
    }
  ]
}