SpamDetection-TorchScript.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b271186f",
   "metadata": {},
   "source": [
    "# Step 1 - Installation of the model required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1ef24c9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: torch in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (1.10.0)\n",
      "Collecting torch\n",
      "  Downloading torch-1.11.0-cp38-cp38-manylinux1_x86_64.whl (750.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m750.6/750.6 MB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from torch) (4.0.0)\n",
      "Installing collected packages: torch\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.10.0\n",
      "    Uninstalling torch-1.10.0:\n",
      "      Successfully uninstalled torch-1.10.0\n",
      "Successfully installed torch-1.11.0\n",
      "\u001b[33mWARNING: You are using pip version 22.0.4; however, version 22.1.2 is available.\n",
      "You should consider upgrading via the '/home/ec2-user/anaconda3/envs/pytorch_p38/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mLooking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: sagemaker in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (2.86.2)\n",
      "Collecting sagemaker\n",
      "  Downloading sagemaker-2.96.0.tar.gz (534 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m534.4/534.4 KB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: attrs==20.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (20.3.0)\n",
      "Requirement already satisfied: boto3<2.0,>=1.20.21 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (1.21.42)\n",
      "Requirement already satisfied: google-pasta in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (1.21.2)\n",
      "Requirement already satisfied: protobuf<4.0,>=3.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (3.19.4)\n",
      "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (0.1.5)\n",
      "Requirement already satisfied: smdebug_rulesconfig==1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (4.8.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (21.3)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (1.3.4)\n",
      "Requirement already satisfied: pathos in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from sagemaker) (0.2.8)\n",
      "Collecting botocore<1.25.0,>=1.24.42\n",
      "  Downloading botocore-1.24.46-py3-none-any.whl (8.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m42.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: s3transfer<0.6.0,>=0.5.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from boto3<2.0,>=1.20.21->sagemaker) (0.5.2)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from boto3<2.0,>=1.20.21->sagemaker) (0.10.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.6.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from packaging>=20.0->sagemaker) (3.0.6)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pandas->sagemaker) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pandas->sagemaker) (2021.3)\n",
      "Requirement already satisfied: pox>=0.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pathos->sagemaker) (0.3.0)\n",
      "Requirement already satisfied: ppft>=1.6.6.4 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pathos->sagemaker) (1.6.6.4)\n",
      "Requirement already satisfied: multiprocess>=0.70.12 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pathos->sagemaker) (0.70.12.2)\n",
      "Requirement already satisfied: dill>=0.3.4 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pathos->sagemaker) (0.3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from botocore<1.25.0,>=1.24.42->boto3<2.0,>=1.20.21->sagemaker) (1.26.8)\n",
      "Building wheels for collected packages: sagemaker\n",
      "  Building wheel for sagemaker (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sagemaker: filename=sagemaker-2.96.0-py2.py3-none-any.whl size=747388 sha256=f72f50936fbd98a9c4207761c4bdd70d199b24ef0af3745889e50c10ec53ab1d\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/85/f0/6e/27188ffde539234dc8ab05d24ec0f29162e84352ac6770b6d6\n",
      "Successfully built sagemaker\n",
      "Installing collected packages: botocore, sagemaker\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.24.19\n",
      "    Uninstalling botocore-1.24.19:\n",
      "      Successfully uninstalled botocore-1.24.19\n",
      "  Attempting uninstall: sagemaker\n",
      "    Found existing installation: sagemaker 2.86.2\n",
      "    Uninstalling sagemaker-2.86.2:\n",
      "      Successfully uninstalled sagemaker-2.86.2\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "awscli 1.22.97 requires botocore==1.24.42, but you have botocore 1.24.46 which is incompatible.\n",
      "aiobotocore 2.0.1 requires botocore<1.22.9,>=1.22.8, but you have botocore 1.24.46 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed botocore-1.24.46 sagemaker-2.96.0\n",
      "\u001b[33mWARNING: You are using pip version 22.0.4; however, version 22.1.2 is available.\n",
      "You should consider upgrading via the '/home/ec2-user/anaconda3/envs/pytorch_p38/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "aiobotocore 2.0.1 requires botocore<1.22.9,>=1.22.8, but you have botocore 1.27.13 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mLooking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Collecting pickle5\n",
      "  Downloading pickle5-0.0.11.tar.gz (132 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.1/132.1 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hBuilding wheels for collected packages: pickle5\n",
      "  Building wheel for pickle5 (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pickle5: filename=pickle5-0.0.11-cp38-cp38-linux_x86_64.whl size=125412 sha256=bee29e07284e1c2ac39711e001f1d1a26c67f0e372e728030ca408063be9a363\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/25/d4/61/dbd8edd1a0d656be7b4267c85db3b61951eb60016a0154a122\n",
      "Successfully built pickle5\n",
      "Installing collected packages: pickle5\n",
      "Successfully installed pickle5-0.0.11\n",
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Collecting datasets\n",
      "  Downloading datasets-2.3.2-py3-none-any.whl (362 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m362.3/362.3 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (1.21.2)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (1.3.4)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (7.0.0)\n",
      "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: dill<0.3.6 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (0.3.4)\n",
      "Requirement already satisfied: aiohttp in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (3.8.1)\n",
      "Collecting xxhash\n",
      "  Downloading xxhash-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.1/212.1 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (0.7.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (2.26.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (4.62.3)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (2021.11.1)\n",
      "Requirement already satisfied: multiprocess in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from datasets) (0.70.12.2)\n",
      "Collecting responses<0.19\n",
      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (5.4.1)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.0.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from packaging->datasets) (3.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.1)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from aiohttp->datasets) (1.7.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from aiohttp->datasets) (20.3.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from aiohttp->datasets) (5.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from pandas->datasets) (2021.3)\n",
      "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.16.0)\n",
      "Installing collected packages: xxhash, responses, datasets\n",
      "Successfully installed datasets-2.3.2 responses-0.18.0 xxhash-3.0.0\n",
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: nltk in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (3.6.5)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from nltk) (2021.11.10)\n",
      "Requirement already satisfied: tqdm in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from nltk) (4.62.3)\n",
      "Requirement already satisfied: click in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from nltk) (8.0.3)\n",
      "Requirement already satisfied: joblib in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from nltk) (1.1.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install torch -U\n",
    "!pip install -U sagemaker\n",
    "!pip install -qU pip awscli boto3 transformers\n",
    "!pip3 install pickle5\n",
    "!pip install datasets\n",
    "!pip install nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ed5ab63",
   "metadata": {},
   "source": [
    "# Step 2 - Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cc00f845",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-21 13:45:13,480 [INFO] Loading the Dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/ec2-user/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-21 13:45:31,658 [INFO] Loading the Metric\n",
      "2022-06-21 13:45:31,726 [INFO] Loading the pretrained tokenizer and model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_transform.bias', 'vocab_transform.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias', 'pre_classifier.weight', 'pre_classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-21 13:45:32,722 [INFO] Preparing the training and evaluation dataset\n",
      "2022-06-21 13:48:07,621 [INFO] Training Started\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 32580\n",
      "  Num Epochs = 4\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 16292\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='16292' max='16292' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [16292/16292 13:27, Epoch 4/4]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.287300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.204600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.201700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.169600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.157700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.175100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>0.147600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>0.168600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>0.097400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>0.114800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5500</td>\n",
       "      <td>0.097800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>0.093500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6500</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>0.097600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7500</td>\n",
       "      <td>0.101700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>0.093500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8500</td>\n",
       "      <td>0.064000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>0.048000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9500</td>\n",
       "      <td>0.042000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>0.057400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10500</td>\n",
       "      <td>0.046200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>0.048600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11500</td>\n",
       "      <td>0.051400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12000</td>\n",
       "      <td>0.036400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12500</td>\n",
       "      <td>0.030900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13000</td>\n",
       "      <td>0.017100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13500</td>\n",
       "      <td>0.024900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14000</td>\n",
       "      <td>0.013800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14500</td>\n",
       "      <td>0.023500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15000</td>\n",
       "      <td>0.019500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15500</td>\n",
       "      <td>0.017000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16000</td>\n",
       "      <td>0.027400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-1000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-1000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-1000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-1500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-1500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-1500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-2000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-2000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-2000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-2500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-2500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-2500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-3000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-3000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-3000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-3500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-3500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-3500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-4000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-4000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-4000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-4500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-4500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-4500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-5000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-5000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-5000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-5500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-5500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-5500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-6000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-6000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-6000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-6500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-6500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-6500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-7000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-7000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-7000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-7500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-7500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-7500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-8000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-8000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-8000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-8500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-8500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-8500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-9000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-9000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-9000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-9500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-9500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-9500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-10000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-10000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-10000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-10500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-10500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-10500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-11000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-11000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-11000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-11500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-11500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-11500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-12000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-12000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-12000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-12500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-12500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-12500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-13000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-13000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-13000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-13500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-13500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-13500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-14000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-14000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-14000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-14500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-14500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-14500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-15000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-15000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-15000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-15500\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-15500/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-15500/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "Saving model checkpoint to ./distilbert_train_intermediate-torchscript/checkpoint-16000\n",
      "Configuration saved in ./distilbert_train_intermediate-torchscript/checkpoint-16000/config.json\n",
      "Model weights saved in ./distilbert_train_intermediate-torchscript/checkpoint-16000/pytorch_model.bin\n",
      "/tmp/ipykernel_20626/2193364873.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Configuration saved in ./workspace-torchscript/config.json\n",
      "Model weights saved in ./workspace-torchscript/pytorch_model.bin\n",
      "tokenizer config file saved in ./workspace-torchscript/tokenizer_config.json\n",
      "Special tokens file saved in ./workspace-torchscript/special_tokens_map.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-21 14:01:38,655 [INFO] Training Completed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 10860\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**************** Evaluation ************\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1358' max='1358' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1358/1358 00:14]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** eval metrics *****\n",
      "  epoch                   =        4.0\n",
      "  eval_accuracy           =     0.9581\n",
      "  eval_loss               =     0.2565\n",
      "  eval_runtime            = 0:00:15.01\n",
      "  eval_samples            =      10860\n",
      "  eval_samples_per_second =    723.116\n",
      "  eval_steps_per_second   =     90.423\n"
     ]
    }
   ],
   "source": [
    "import pickle5 as pickle\n",
    "import logging\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import nltk\n",
    "from sklearn.model_selection import train_test_split\n",
    "from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, Trainer, TrainingArguments\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import sys\n",
    "from datasets import load_metric\n",
    "import numpy as np\n",
    "\n",
    "TRAIN_DIR = \"./distilbert_train_intermediate-torchscript\"\n",
    "FINAL_DIR = \"./workspace-torchscript\"\n",
    "DEFAULT_FILENAME = \"./spam_training_dataset_43k.pkl\"\n",
    "BATCH_SIZE = 128\n",
    "COL_DATA = \"text\"  # Name of the column with the spam text\n",
    "LABEL = \"is_spam\"  # Name of the column with the label 0 (ham) or 1 (spam)\n",
    "NUM_EPOCHS = 4\n",
    "\n",
    "logging.basicConfig(\n",
    "    level=logging.INFO,\n",
    "    format=\"%(asctime)s [%(levelname)s] %(message)s\",\n",
    "    handlers=[\n",
    "        logging.StreamHandler(sys.stdout)\n",
    "    ]\n",
    ")\n",
    "\n",
    "def clean_text(text):\n",
    "    import re\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "    text = str(text).lower()  # Convert to lower case\n",
    "    text = re.sub(r'[^\\w\\s]', '', text)  # Remove everything except words\n",
    "    words = [word for word in text.split() if word not in stopwords]  # Remove stopwords\n",
    "    text = \" \".join(words)\n",
    "    return text\n",
    "\n",
    "def download_dataset():  \n",
    "    nltk.download(\"stopwords\")\n",
    "\n",
    "    with open(DEFAULT_FILENAME, \"rb\") as fh:\n",
    "        data = pickle.load(fh)\n",
    "        data = data[[COL_DATA, LABEL]]  \n",
    "        data[COL_DATA] = data[COL_DATA].apply(clean_text)\n",
    "        data.reset_index()\n",
    "        return data\n",
    "\n",
    "class ClassificationDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)    \n",
    "    \n",
    "logging.info('Loading the Dataset')\n",
    "dataset = download_dataset()\n",
    "\n",
    "logging.info('Loading the Metric')\n",
    "metric = load_metric(\"accuracy\")\n",
    "\n",
    "logging.info('Loading the pretrained tokenizer and model')\n",
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "model = DistilBertForSequenceClassification.from_pretrained(\"distilbert-base-uncased\")\n",
    "\n",
    "logging.info('Preparing the training and evaluation dataset')\n",
    "train_data, val_data, train_labels, val_labels = train_test_split(dataset[COL_DATA].values, dataset[LABEL].values)\n",
    "train_tokens = tokenizer(list(train_data), return_tensors=\"pt\", padding=True, truncation=True, max_length=BATCH_SIZE)\n",
    "val_tokens = tokenizer(list(val_data), return_tensors=\"pt\", padding=True, truncation=True, max_length=BATCH_SIZE)\n",
    "\n",
    "train_dataset = ClassificationDataset(train_tokens, train_labels)\n",
    "val_dataset = ClassificationDataset(val_tokens, val_labels)\n",
    "\n",
    "logging.info('Training Started')\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=TrainingArguments(output_dir=TRAIN_DIR, num_train_epochs=NUM_EPOCHS),\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "\n",
    "model.save_pretrained(FINAL_DIR)\n",
    "tokenizer.save_pretrained(FINAL_DIR)\n",
    "logging.info('Training Completed')\n",
    "\n",
    "print(\"**************** Evaluation ************\")\n",
    "metrics = trainer.evaluate()\n",
    "metrics[\"eval_samples\"] = len(val_dataset)\n",
    "trainer.log_metrics(\"eval\", metrics)\n",
    "trainer.save_metrics(\"eval\", metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "952a2c44",
   "metadata": {},
   "source": [
    "# Step 3 - Generate the PT file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8778ad8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file ./workspace-torchscript/config.json\n",
      "Model config DistilBertConfig {\n",
      "  \"_name_or_path\": \"distilbert-base-uncased\",\n",
      "  \"activation\": \"gelu\",\n",
      "  \"architectures\": [\n",
      "    \"DistilBertForSequenceClassification\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"dim\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"distilbert\",\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"problem_type\": \"single_label_classification\",\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"tie_weights_\": true,\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"torchscript\": true,\n",
      "  \"transformers_version\": \"4.20.0\",\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file ./workspace-torchscript/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing DistilBertForSequenceClassification.\n",
      "\n",
      "All the weights of DistilBertForSequenceClassification were initialized from the model checkpoint at ./workspace-torchscript/.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForSequenceClassification for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, Trainer, TrainingArguments\n",
    "import torch\n",
    "\n",
    "loaded_model = DistilBertForSequenceClassification.from_pretrained(\"./workspace-torchscript/\", torchscript=True)\n",
    "device = torch.device(\"cuda\")\n",
    "loaded_model.to(device)\n",
    "\n",
    "bs = 1\n",
    "seq_len = 128\n",
    "dummy_inputs = [\n",
    "    torch.randint(1000, (bs, seq_len)).to(device),\n",
    "    torch.zeros(bs, seq_len, dtype=torch.int).to(device),\n",
    "]\n",
    "\n",
    "loaded_model = loaded_model.eval()\n",
    "loaded_model.to(device)\n",
    "\n",
    "traced_model = torch.jit.trace(loaded_model, dummy_inputs)\n",
    "torch.jit.save(traced_model, './spamdetection-torchscript/model/model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "458af818",
   "metadata": {},
   "source": [
    "# Step 4 - Create SageMaker model package and upload it to SageMaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "3e98f384",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.pt\n"
     ]
    }
   ],
   "source": [
    "!cd spamdetection-torchscript/model && tar czvf ../model.tar.gz *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db2111ca",
   "metadata": {},
   "source": [
    "# Step 5 - Create SageMaker Inference endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "b8ac8016",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://sagemaker-us-east-1-806460758762/spamdetection-torchscript/model.tar.gz\n",
      "2022-06-21 16:32:41,690 [INFO] Creating model with name: spamdetection-torchscript-2022-06-21-16-31-54\n",
      "2022-06-21 16:32:41,975 [INFO] Creating endpoint-config with name spamdetection-torchscript-2022-06-21-16-31-54\n",
      "2022-06-21 16:32:42,050 [INFO] Creating endpoint with name spamdetection-torchscript-2022-06-21-16-31-54\n",
      "------------!"
     ]
    }
   ],
   "source": [
    "import boto3, json, sagemaker, time\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.pytorch import PyTorchModel\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.serializers import JSONSerializer\n",
    "from sagemaker.deserializers import JSONDeserializer\n",
    "\n",
    "sess = boto3.Session()\n",
    "sm = sess.client(\"sagemaker\")\n",
    "sagemaker_session = sagemaker.Session(boto_session=sess)\n",
    "role = get_execution_role()\n",
    "\n",
    "model_uri = sagemaker_session.upload_data(path=\"./spamdetection-torchscript/model.tar.gz\", key_prefix=\"spamdetection-torchscript\")\n",
    "print(model_uri)\n",
    "\n",
    "sm_model_name = \"spamdetection-torchscript-\" + time.strftime(\"%Y-%m-%d-%H-%M-%S\", time.gmtime())\n",
    "\n",
    "model = PyTorchModel(\n",
    "              role=role,\n",
    "              name=sm_model_name,\n",
    "              sagemaker_session=sagemaker_session,\n",
    "              model_data=model_uri,\n",
    "              framework_version='1.11.0',\n",
    "              py_version='py38',\n",
    "              entry_point=\"serve.py\",\n",
    "              source_dir=\"spamdetection-torchscript\",\n",
    "              )\n",
    "\n",
    "predictor = model.deploy(\n",
    "    initial_instance_count=1,\n",
    "    instance_type=\"ml.p3.2xlarge\",\n",
    "    endpoint_name=sm_model_name,\n",
    "    serializer=JSONSerializer(),\n",
    "    deserializer=JSONDeserializer(),    \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9ab8a8d",
   "metadata": {},
   "source": [
    "# Step 6 - Test Inference Point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0fecd226",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Collecting retry\n",
      "  Downloading retry-0.9.2-py2.py3-none-any.whl (8.0 kB)\n",
      "Requirement already satisfied: py<2.0.0,>=1.4.26 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from retry) (1.11.0)\n",
      "Requirement already satisfied: decorator>=3.4.2 in /home/ec2-user/anaconda3/envs/pytorch_p38/lib/python3.8/site-packages (from retry) (5.1.0)\n",
      "Installing collected packages: retry\n",
      "Successfully installed retry-0.9.2\n"
     ]
    }
   ],
   "source": [
    "!pip install retry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "80461a99",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Didn't find file ./workspace-torchscript/added_tokens.json. We won't load it.\n",
      "loading file ./workspace-torchscript/vocab.txt\n",
      "loading file None\n",
      "loading file ./workspace-torchscript/special_tokens_map.json\n",
      "loading file ./workspace-torchscript/tokenizer_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-21 16:59:41,934 [WARNING] Connection pool is full, discarding connection: runtime.sagemaker.us-east-1.amazonaws.com. Connection pool size: 10\n",
      "2022-06-21 16:59:41,941 [WARNING] Connection pool is full, discarding connection: runtime.sagemaker.us-east-1.amazonaws.com. Connection pool size: 10\n",
      "num_inferences:  4000[texts], elapsed_time: 28.49[sec], Throughput:  140.40[texts/sec]\n"
     ]
    }
   ],
   "source": [
    "from retry import retry\n",
    "import botocore\n",
    "import concurrent\n",
    "import torch.nn.functional as F    \n",
    "\n",
    "def tokenize_text(text):\n",
    "    encoded_text = enc(clean_text(text), padding=\"max_length\", max_length=128, truncation=True)\n",
    "    return encoded_text[\"input_ids\"], encoded_text[\"attention_mask\"]\n",
    "\n",
    "@retry(botocore.exceptions.ClientError, tries=5, delay=1)\n",
    "def get_prediction(test_text):\n",
    "    input_ids, attention_mask = tokenize_text(test_text)\n",
    "    result = predictor.predict({\"input_ids\" : input_ids, \"attention_mask\" : attention_mask}, initial_args={'ContentType': 'application/json'})\n",
    "    predictions = F.softmax(torch.tensor(result[0]),dim=-1)\n",
    "    return torch.argmax(predictions, dim=-1).numpy()\n",
    "\n",
    "enc = DistilBertTokenizer.from_pretrained(\"./workspace-torchscript\")\n",
    "\n",
    "test_texts = [\n",
    "                \"Oh k...i'''m watching here:)\",\n",
    "                \"As a valued customer, I am pleased to advise you that following recent review of your Mob No. you are awarded with a £1500 Bonus Prize, call 09066364589\",\n",
    "                \"I HAVE A DATE ON SUNDAY WITH WILL!!\",\n",
    "                \"England v Macedonia - dont miss the goals/team news. Txt ur national team to 87077 eg ENGLAND to 87077 Try:WALES, SCOTLAND 4txt/ú1.20 POBOXox36504W45WQ 16+\"\n",
    "]\n",
    "\n",
    "num_inferences = 1000\n",
    "start = time.time() \n",
    "with concurrent.futures.ThreadPoolExecutor() as exe: \n",
    "    fut_list = []\n",
    "    for _ in range (num_inferences):\n",
    "        for test_text in test_texts:\n",
    "            fut = exe.submit(get_prediction, test_text)         \n",
    "            fut_list.append(fut)     \n",
    "    for fut in fut_list:         \n",
    "        rslt = fut.result() \n",
    "        \n",
    "elapsed_time = time.time() - start \n",
    "print('num_inferences:{:>6}[texts], elapsed_time:{:6.2f}[sec], Throughput:{:8.2f}[texts/sec]'.format(num_inferences * len(test_texts), elapsed_time, num_inferences * len(test_texts)/ elapsed_time))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d053421",
   "metadata": {},
   "source": [
    "# Step 7 - Delete the Inference Endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91acfc4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor.delete_endpoint()\n",
    "predictor.delete_model(ModelName=sm_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba40fe2b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p38",
   "language": "python",
   "name": "conda_pytorch_p38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}