05.Model Training.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "aq7EGYlCna-v"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bHmCk5LpngV5",
        "outputId": "9c867e04-49c4-466b-8100-ed792bf3d07f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_af_zBLgngUM",
        "outputId": "2d8abdfe-7864-423c-b629-1b97e4c4b391"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/BERT-NER\n"
          ]
        }
      ],
      "source": [
        "%cd /content/drive/MyDrive/BERT-NER"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GsoH1UWongSW",
        "outputId": "47e8b138-23b7-4c26-a8ac-3170d1b673c5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "data  models\n"
          ]
        }
      ],
      "source": [
        "!ls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FMaHh6v4ngQS",
        "outputId": "9aed870f-a2fc-426f-98df-62dbac6525bf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: flair in /usr/local/lib/python3.10/dist-packages (0.12.2)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.10/dist-packages (from flair) (2.8.2)\n",
            "Requirement already satisfied: torch!=1.8,>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from flair) (2.0.0)\n",
            "Requirement already satisfied: gensim>=3.8.0 in /usr/local/lib/python3.10/dist-packages (from flair) (4.3.2)\n",
            "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.10/dist-packages (from flair) (4.66.1)\n",
            "Requirement already satisfied: segtok>=1.5.7 in /usr/local/lib/python3.10/dist-packages (from flair) (1.5.11)\n",
            "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.10/dist-packages (from flair) (3.7.1)\n",
            "Requirement already satisfied: mpld3==0.3 in /usr/local/lib/python3.10/dist-packages (from flair) (0.3)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from flair) (1.2.2)\n",
            "Requirement already satisfied: sqlitedict>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from flair) (2.1.0)\n",
            "Requirement already satisfied: deprecated>=1.2.4 in /usr/local/lib/python3.10/dist-packages (from flair) (1.2.14)\n",
            "Requirement already satisfied: hyperopt>=0.2.7 in /usr/local/lib/python3.10/dist-packages (from flair) (0.2.7)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.10/dist-packages (from flair) (1.28.53)\n",
            "Requirement already satisfied: transformers[sentencepiece]>=4.18.0 in /usr/local/lib/python3.10/dist-packages (from flair) (4.33.2)\n",
            "Requirement already satisfied: bpemb>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from flair) (0.3.4)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from flair) (2023.6.3)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from flair) (0.9.0)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.10/dist-packages (from flair) (1.0.9)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from flair) (4.9.3)\n",
            "Requirement already satisfied: ftfy in /usr/local/lib/python3.10/dist-packages (from flair) (6.1.1)\n",
            "Requirement already satisfied: janome in /usr/local/lib/python3.10/dist-packages (from flair) (0.5.0)\n",
            "Requirement already satisfied: gdown==4.4.0 in /usr/local/lib/python3.10/dist-packages (from flair) (4.4.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from flair) (0.17.2)\n",
            "Requirement already satisfied: conllu>=4.0 in /usr/local/lib/python3.10/dist-packages (from flair) (4.5.3)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from flair) (10.1.0)\n",
            "Requirement already satisfied: wikipedia-api in /usr/local/lib/python3.10/dist-packages (from flair) (0.6.0)\n",
            "Requirement already satisfied: pptree in /usr/local/lib/python3.10/dist-packages (from flair) (3.1)\n",
            "Requirement already satisfied: pytorch-revgrad in /usr/local/lib/python3.10/dist-packages (from flair) (0.2.0)\n",
            "Requirement already satisfied: transformer-smaller-training-vocab>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from flair) (0.3.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from gdown==4.4.0->flair) (3.12.2)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.10/dist-packages (from gdown==4.4.0->flair) (2.31.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from gdown==4.4.0->flair) (1.16.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from gdown==4.4.0->flair) (4.11.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bpemb>=0.3.2->flair) (1.23.5)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from bpemb>=0.3.2->flair) (0.1.99)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.4->flair) (1.15.0)\n",
            "Requirement already satisfied: scipy>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from gensim>=3.8.0->flair) (1.11.2)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim>=3.8.0->flair) (6.4.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.10.0->flair) (2023.6.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.10.0->flair) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.10.0->flair) (4.5.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.10.0->flair) (23.1)\n",
            "Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.10/dist-packages (from hyperopt>=0.2.7->flair) (3.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from hyperopt>=0.2.7->flair) (0.18.3)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from hyperopt>=0.2.7->flair) (2.2.1)\n",
            "Requirement already satisfied: py4j in /usr/local/lib/python3.10/dist-packages (from hyperopt>=0.2.7->flair) (0.10.9.7)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.2.3->flair) (1.1.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.2.3->flair) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.2.3->flair) (4.42.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.2.3->flair) (1.4.5)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.2.3->flair) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.2.3->flair) (3.1.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->flair) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->flair) (3.2.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (1.12)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (3.1.2)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.7.99)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.7.99)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.7.101)\n",
            "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (8.5.0.96)\n",
            "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.10.3.66)\n",
            "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (10.9.0.58)\n",
            "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (10.2.10.91)\n",
            "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.4.0.1)\n",
            "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.7.4.91)\n",
            "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (2.14.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (11.7.91)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.5.0->flair) (2.0.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch!=1.8,>=1.5.0->flair) (67.7.2)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch!=1.8,>=1.5.0->flair) (0.41.2)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.8,>=1.5.0->flair) (3.27.4.1)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.8,>=1.5.0->flair) (16.0.6)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.18.0->flair) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.18.0->flair) (0.3.3)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.18.0->flair) (3.20.3)\n",
            "Requirement already satisfied: botocore<1.32.0,>=1.31.53 in /usr/local/lib/python3.10/dist-packages (from boto3->flair) (1.31.53)\n",
            "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3->flair) (1.0.1)\n",
            "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from boto3->flair) (0.6.2)\n",
            "Requirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.10/dist-packages (from ftfy->flair) (0.2.6)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.32.0,>=1.31.53->boto3->flair) (1.26.16)\n",
            "Requirement already satisfied: accelerate>=0.20.3 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.18.0->flair) (0.23.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->gdown==4.4.0->flair) (2.5)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.8,>=1.5.0->flair) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==4.4.0->flair) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==4.4.0->flair) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==4.4.0->flair) (2023.7.22)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==4.4.0->flair) (1.7.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.8,>=1.5.0->flair) (1.3.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[sentencepiece]>=4.18.0->flair) (5.9.5)\n"
          ]
        }
      ],
      "source": [
        "# install required dependencies\n",
        "\n",
        "!pip3 install flair"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction import DictVectorizer\n",
        "from sklearn.feature_extraction.text import HashingVectorizer\n",
        "from sklearn.linear_model import Perceptron\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.linear_model import PassiveAggressiveClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.metrics import classification_report\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "2GJVhTPIiica"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load train, validation, test word label datasets\n",
        "\n",
        "train_ds = pd.read_csv('data/ner/train_word_label.csv')\n",
        "train_ds"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "a_8uH8veiiaY",
        "outputId": "bbb871c1-b58f-4e98-a727-92d015fd024b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             word       label\n",
              "0            Good           O\n",
              "1          catch.           O\n",
              "2              NY  B-LOCATION\n",
              "3             law           O\n",
              "4          should           O\n",
              "...           ...         ...\n",
              "72855  voicemail.           O\n",
              "72856       Thank           O\n",
              "72857        you.           O\n",
              "72858       Karen    B-PERSON\n",
              "72859      x39757           O\n",
              "\n",
              "[72860 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f94b49a4-4594-4749-b1ac-482e1947a72e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Good</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>catch.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NY</td>\n",
              "      <td>B-LOCATION</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>law</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>should</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72855</th>\n",
              "      <td>voicemail.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72856</th>\n",
              "      <td>Thank</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72857</th>\n",
              "      <td>you.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72858</th>\n",
              "      <td>Karen</td>\n",
              "      <td>B-PERSON</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72859</th>\n",
              "      <td>x39757</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>72860 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f94b49a4-4594-4749-b1ac-482e1947a72e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f94b49a4-4594-4749-b1ac-482e1947a72e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f94b49a4-4594-4749-b1ac-482e1947a72e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5727aa71-1bcc-4fcb-b6c8-4a2896fd998d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5727aa71-1bcc-4fcb-b6c8-4a2896fd998d')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5727aa71-1bcc-4fcb-b6c8-4a2896fd998d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_ds.fillna(0, inplace=True)"
      ],
      "metadata": {
        "id": "VOWYbWXMiiRu"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "val_ds = pd.read_csv('data/ner/val_word_label.csv')\n",
        "val_ds"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "OfiwQlAliiPc",
        "outputId": "9f6ff8f6-ebc2-4d16-db62-8c70644165b2"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           word     label\n",
              "0            Al  B-PERSON\n",
              "1             :         O\n",
              "2           Per         O\n",
              "3            my         O\n",
              "4         voice         O\n",
              "...         ...       ...\n",
              "9077   database         O\n",
              "9078          !         O\n",
              "9079   !Unknown         O\n",
              "9080  database.         O\n",
              "9081      Alias         O\n",
              "\n",
              "[9082 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1e028d93-02bf-441a-844e-8c944819ced4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Al</td>\n",
              "      <td>B-PERSON</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>:</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Per</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>my</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>voice</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9077</th>\n",
              "      <td>database</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9078</th>\n",
              "      <td>!</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9079</th>\n",
              "      <td>!Unknown</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9080</th>\n",
              "      <td>database.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9081</th>\n",
              "      <td>Alias</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>9082 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1e028d93-02bf-441a-844e-8c944819ced4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-1e028d93-02bf-441a-844e-8c944819ced4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-1e028d93-02bf-441a-844e-8c944819ced4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5c0d8aa8-7f1f-468d-973e-13514c38800c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5c0d8aa8-7f1f-468d-973e-13514c38800c')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5c0d8aa8-7f1f-468d-973e-13514c38800c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_ds = pd.read_csv('data/ner/test_word_label.csv')\n",
        "test_ds"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "Tpw79-wQiiNV",
        "outputId": "4bc330a1-11bc-499a-ebc4-3742a2bf1f8e"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             word     label\n",
              "0        Allright         O\n",
              "1          fellas         O\n",
              "2            here         O\n",
              "3              's         O\n",
              "4             the         O\n",
              "...           ...       ...\n",
              "8985  information         O\n",
              "8986          was         O\n",
              "8987     supplied         O\n",
              "8988          me.         O\n",
              "8989       Stuart  B-PERSON\n",
              "\n",
              "[8990 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-288f08a2-c061-4ad3-a2b1-6f8fcc286de1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Allright</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>fellas</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>here</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>'s</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>the</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8985</th>\n",
              "      <td>information</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8986</th>\n",
              "      <td>was</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8987</th>\n",
              "      <td>supplied</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8988</th>\n",
              "      <td>me.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8989</th>\n",
              "      <td>Stuart</td>\n",
              "      <td>B-PERSON</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8990 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-288f08a2-c061-4ad3-a2b1-6f8fcc286de1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-288f08a2-c061-4ad3-a2b1-6f8fcc286de1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-288f08a2-c061-4ad3-a2b1-6f8fcc286de1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-69b6fbb2-6add-42f9-8dde-c4aba0ef5378\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-69b6fbb2-6add-42f9-8dde-c4aba0ef5378')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-69b6fbb2-6add-42f9-8dde-c4aba0ef5378 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_ds.fillna(0, inplace=True)"
      ],
      "metadata": {
        "id": "PEMafwD_jo4U"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert to numerical format train, test words\n",
        "\n",
        "v = DictVectorizer(sparse=False)\n",
        "\n",
        "X_train = v.fit_transform(\n",
        "    train_ds.drop('label', axis=1).to_dict(orient='records')\n",
        ")\n",
        "X_test = v.transform(\n",
        "    test_ds.drop('label', axis=1).to_dict(orient='records')\n",
        ")"
      ],
      "metadata": {
        "id": "rC2fE8sajqIt"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dOo7qb9sjqGo",
        "outputId": "dd5c144b-8b43-432f-9f62-25d8f8567fa8"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_train = train_ds.label.values\n",
        "y_train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lBIUQVTUjqC2",
        "outputId": "0df4caea-8059-4220-c506-ddea41fd32b2"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['O', 'O', 'B-LOCATION', ..., 'O', 'B-PERSON', 'O'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vOxEjVSakCJE",
        "outputId": "ade79f2d-1a5e-4e52-ee2d-1c4ea0ab3acc"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_test = test_ds.label.values\n",
        "y_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cPprbBU5kCG-",
        "outputId": "a6ef5527-467f-4b45-d052-7482ad1a1f1c"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['O', 'O', 'O', ..., 'O', 'O', 'B-PERSON'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_test.shape, y_test.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HzSBGUr2kCEo",
        "outputId": "a7ebc7d7-ed62-4939-f4d4-b946ba4e037f"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((8990, 16230), (8990,))"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll use few methods to determine the baseline model.\n",
        "Afterwards we'll compare with BERT for NER task."
      ],
      "metadata": {
        "id": "bO6jNpVTkYky"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Determining Baseline Model**"
      ],
      "metadata": {
        "id": "Wamc5nKgkbgp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create utility function to store evaluation metrics\n",
        "\n",
        "# Performance metrics\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "models_dct = {\n",
        "    'model_type': [],\n",
        "    'precision': [],\n",
        "    'recall': [],\n",
        "    'f1_score': []\n",
        "}\n",
        "\n",
        "def evaluate_model(model, test_labels, labels, model_type):\n",
        "    \"\"\"\n",
        "    Evaluate model\n",
        "    \"\"\"\n",
        "    y_pred = model.predict(X_test)\n",
        "\n",
        "    report = classification_report(\n",
        "        y_pred=y_pred, y_true=y_test, labels=labels, output_dict=True)\n",
        "\n",
        "    _p_score = report['macro avg']['precision']\n",
        "    _r_score = report['macro avg']['recall']\n",
        "    _f1_score = report['macro avg']['f1-score']\n",
        "\n",
        "    models_dct['model_type'].append(model_type)\n",
        "    models_dct['precision'].append(_p_score)\n",
        "    models_dct['recall'].append(_r_score)\n",
        "    models_dct['f1_score'].append(_f1_score)"
      ],
      "metadata": {
        "id": "FIEkTLG1kat6"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Perceptron**"
      ],
      "metadata": {
        "id": "PUkCEe-zkjvO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "all_labels = train_ds['label'].values\n",
        "classes = np.unique(all_labels)\n",
        "classes = classes.tolist()\n",
        "classes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YYVfvXZfkaoo",
        "outputId": "aaa54883-ca84-41e3-c09a-9438446ca5d0"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['B-DATE',\n",
              " 'B-LOCATION',\n",
              " 'B-ORG',\n",
              " 'B-PERSON',\n",
              " 'I-DATE',\n",
              " 'I-LOCATION',\n",
              " 'I-ORG',\n",
              " 'I-PERSON',\n",
              " 'O']"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Because the tag \"O\" (outside) is the most common tag, will make our results look much better than they actual are.\n",
        "# Therefore we remove the tag \"O\" when we evaluate classification metrics.\n",
        "\n",
        "new_classes = classes.copy()\n",
        "new_classes.pop()\n",
        "new_classes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_tHQZyVfkimC",
        "outputId": "9825cd72-e041-4e36-90d2-bf1adf4fc919"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['B-DATE',\n",
              " 'B-LOCATION',\n",
              " 'B-ORG',\n",
              " 'B-PERSON',\n",
              " 'I-DATE',\n",
              " 'I-LOCATION',\n",
              " 'I-ORG',\n",
              " 'I-PERSON']"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "per = Perceptron(verbose=10, n_jobs=-1, max_iter=5)\n",
        "per.partial_fit(X_train, y_train, classes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 631
        },
        "id": "PdgO2S9Dkijp",
        "outputId": "5280ee7b-6819-4164-b4a6-f7fbc94b15d1"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 2 concurrent workers.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-- Epoch 1\n",
            "-- Epoch 1\n",
            "Norm: 23.30, NNZs: 444, Bias: -1.000000, T: 72860, Avg. loss: 0.007123\n",
            "Total training time: 4.31 seconds.\n",
            "-- Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:    4.3s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Norm: 15.23, NNZs: 192, Bias: -2.000000, T: 72860, Avg. loss: 0.007878\n",
            "Total training time: 4.61 seconds.\n",
            "-- Epoch 1\n",
            "Norm: 34.38, NNZs: 944, Bias: -2.000000, T: 72860, Avg. loss: 0.012970\n",
            "Total training time: 4.57 seconds.\n",
            "-- Epoch 1\n",
            "Norm: 48.63, NNZs: 1554, Bias: -1.000000, T: 72860, Avg. loss: 0.013780\n",
            "Total training time: 4.76 seconds.\n",
            "-- Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Done   4 tasks      | elapsed:    9.4s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Norm: 16.37, NNZs: 198, Bias: -2.000000, T: 72860, Avg. loss: 0.009360\n",
            "Total training time: 3.01 seconds.\n",
            "-- Epoch 1\n",
            "Norm: 18.87, NNZs: 317, Bias: -2.000000, T: 72860, Avg. loss: 0.004968\n",
            "Total training time: 2.67 seconds.\n",
            "-- Epoch 1\n",
            "Norm: 26.98, NNZs: 541, Bias: -2.000000, T: 72860, Avg. loss: 0.011639\n",
            "Total training time: 2.53 seconds.\n",
            "-- Epoch 1\n",
            "Norm: 37.80, NNZs: 1142, Bias: -1.000000, T: 72860, Avg. loss: 0.010925\n",
            "Total training time: 2.49 seconds.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Done   7 out of   9 | elapsed:   14.5s remaining:    4.1s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Norm: 86.22, NNZs: 5012, Bias: 0.000000, T: 72860, Avg. loss: 0.066772\n",
            "Total training time: 2.06 seconds.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Done   9 out of   9 | elapsed:   16.5s finished\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Perceptron(max_iter=5, n_jobs=-1, verbose=10)"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Perceptron(max_iter=5, n_jobs=-1, verbose=10)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Perceptron</label><div class=\"sk-toggleable__content\"><pre>Perceptron(max_iter=5, n_jobs=-1, verbose=10)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_model(per, y_test, new_classes, 'Perceptron')"
      ],
      "metadata": {
        "id": "UI-87Njlkihj"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "models_dct"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "izGJGqvXkifq",
        "outputId": "a1f328b3-5613-4a56-91c9-727fffd4bf77"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'model_type': ['Perceptron'],\n",
              " 'precision': [0.45005032404262235],\n",
              " 'recall': [0.30770586146201906],\n",
              " 'f1_score': [0.32818991528950503]}"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Output classification report as dictionary to extract F1-score\n",
        "\n",
        "report = classification_report(\n",
        "    y_pred=per.predict(X_test), y_true=y_test, labels=new_classes, zero_division=0, output_dict=True\n",
        ")\n",
        "report"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V1rICkq2kidm",
        "outputId": "0142d74d-2ea0-45f6-dea0-f5d570ee7d45"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'B-DATE': {'precision': 0.2391304347826087,\n",
              "  'recall': 0.22448979591836735,\n",
              "  'f1-score': 0.23157894736842108,\n",
              "  'support': 49},\n",
              " 'B-LOCATION': {'precision': 0.1484375,\n",
              "  'recall': 0.48717948717948717,\n",
              "  'f1-score': 0.22754491017964068,\n",
              "  'support': 78},\n",
              " 'B-ORG': {'precision': 0.7121212121212122,\n",
              "  'recall': 0.3560606060606061,\n",
              "  'f1-score': 0.47474747474747475,\n",
              "  'support': 132},\n",
              " 'B-PERSON': {'precision': 0.7747252747252747,\n",
              "  'recall': 0.6714285714285714,\n",
              "  'f1-score': 0.7193877551020408,\n",
              "  'support': 210},\n",
              " 'I-DATE': {'precision': 0.3684210526315789,\n",
              "  'recall': 0.11666666666666667,\n",
              "  'f1-score': 0.17721518987341772,\n",
              "  'support': 60},\n",
              " 'I-LOCATION': {'precision': 0.3333333333333333,\n",
              "  'recall': 0.047619047619047616,\n",
              "  'f1-score': 0.08333333333333333,\n",
              "  'support': 42},\n",
              " 'I-ORG': {'precision': 0.5652173913043478,\n",
              "  'recall': 0.23636363636363636,\n",
              "  'f1-score': 0.3333333333333333,\n",
              "  'support': 110},\n",
              " 'I-PERSON': {'precision': 0.45901639344262296,\n",
              "  'recall': 0.3218390804597701,\n",
              "  'f1-score': 0.3783783783783784,\n",
              "  'support': 87},\n",
              " 'micro avg': {'precision': 0.4398826979472141,\n",
              "  'recall': 0.390625,\n",
              "  'f1-score': 0.4137931034482758,\n",
              "  'support': 768},\n",
              " 'macro avg': {'precision': 0.45005032404262235,\n",
              "  'recall': 0.30770586146201906,\n",
              "  'f1-score': 0.32818991528950503,\n",
              "  'support': 768},\n",
              " 'weighted avg': {'precision': 0.5445331073275218,\n",
              "  'recall': 0.390625,\n",
              "  'f1-score': 0.42519852035306055,\n",
              "  'support': 768}}"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Passive Aggresive Classifier**"
      ],
      "metadata": {
        "id": "lcpG6NFXlBd6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pa = PassiveAggressiveClassifier()\n",
        "pa.partial_fit(X_train, y_train, classes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 75
        },
        "id": "1X0TaE5Kk2lN",
        "outputId": "09eecf3a-6784-4a00-9fcd-c1717a75d420"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PassiveAggressiveClassifier()"
            ],
            "text/html": [
              "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>PassiveAggressiveClassifier()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">PassiveAggressiveClassifier</label><div class=\"sk-toggleable__content\"><pre>PassiveAggressiveClassifier()</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_model(pa, y_test, new_classes, 'Passive Aggresive Classifier')"
      ],
      "metadata": {
        "id": "TmLZ6GeDk2iZ"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Naive Bayes Classifier**"
      ],
      "metadata": {
        "id": "0a1R575_lIsF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nb = MultinomialNB(alpha=0.01)\n",
        "nb.partial_fit(X_train, y_train, classes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 75
        },
        "id": "hYoQj0hrk2gO",
        "outputId": "abea0e30-7882-434d-badd-a8cc96b19b05"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MultinomialNB(alpha=0.01)"
            ],
            "text/html": [
              "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB(alpha=0.01)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB(alpha=0.01)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_model(nb, y_test, new_classes, 'Naive Bayes Classifier')"
      ],
      "metadata": {
        "id": "OOEWAzQ9k2d-"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Liniar SGD Classifier**"
      ],
      "metadata": {
        "id": "XIsyateblT4l"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sgd = SGDClassifier()\n",
        "sgd.partial_fit(X_train, y_train, classes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 75
        },
        "id": "QrX16uEQlTpU",
        "outputId": "a48a4823-ac09-48d3-e063-1de4edad57cb"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SGDClassifier()"
            ],
            "text/html": [
              "<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SGDClassifier()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SGDClassifier</label><div class=\"sk-toggleable__content\"><pre>SGDClassifier()</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_model(sgd, y_test, new_classes, 'SGD Classifier')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GoYUTLV4lNmy",
        "outputId": "369242c1-b0b8-403c-abda-417b8752aee8"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "models_dct"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q3qeqNAFlNkL",
        "outputId": "a13327e6-7577-48ce-9ccf-ebd85cd47347"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'model_type': ['Perceptron',\n",
              "  'Passive Aggresive Classifier',\n",
              "  'Naive Bayes Classifier',\n",
              "  'SGD Classifier'],\n",
              " 'precision': [0.45005032404262235,\n",
              "  0.4914428713668629,\n",
              "  0.4882471350436121,\n",
              "  0.4213529936186186],\n",
              " 'recall': [0.30770586146201906,\n",
              "  0.3114502082482378,\n",
              "  0.267575579182722,\n",
              "  0.09009603901845281],\n",
              " 'f1_score': [0.32818991528950503,\n",
              "  0.36177860495555403,\n",
              "  0.33272083494314847,\n",
              "  0.14333242705137994]}"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eval_df = pd.DataFrame(models_dct)\n",
        "eval_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "id": "tHltXgiAlNeA",
        "outputId": "141ae2ca-0b45-4400-ab6a-a597da8b7961"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                     model_type  precision    recall  f1_score\n",
              "0                    Perceptron   0.450050  0.307706  0.328190\n",
              "1  Passive Aggresive Classifier   0.491443  0.311450  0.361779\n",
              "2        Naive Bayes Classifier   0.488247  0.267576  0.332721\n",
              "3                SGD Classifier   0.421353  0.090096  0.143332"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9c46cf4c-1043-4739-ad2d-a79e2ddbf972\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model_type</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Perceptron</td>\n",
              "      <td>0.450050</td>\n",
              "      <td>0.307706</td>\n",
              "      <td>0.328190</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Passive Aggresive Classifier</td>\n",
              "      <td>0.491443</td>\n",
              "      <td>0.311450</td>\n",
              "      <td>0.361779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Naive Bayes Classifier</td>\n",
              "      <td>0.488247</td>\n",
              "      <td>0.267576</td>\n",
              "      <td>0.332721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>SGD Classifier</td>\n",
              "      <td>0.421353</td>\n",
              "      <td>0.090096</td>\n",
              "      <td>0.143332</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9c46cf4c-1043-4739-ad2d-a79e2ddbf972')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-9c46cf4c-1043-4739-ad2d-a79e2ddbf972 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-9c46cf4c-1043-4739-ad2d-a79e2ddbf972');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-94e93078-369f-4e67-a5aa-5437517bea15\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-94e93078-369f-4e67-a5aa-5437517bea15')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-94e93078-369f-4e67-a5aa-5437517bea15 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_types = models_dct['model_type']\n",
        "model_types"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EtJ05AOalkOg",
        "outputId": "b5bb5f22-33dc-4ac5-a01c-28d40a61f22b"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Perceptron',\n",
              " 'Passive Aggresive Classifier',\n",
              " 'Naive Bayes Classifier',\n",
              " 'SGD Classifier']"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make model types as dataframe index\n",
        "\n",
        "eval_df.index = model_types\n",
        "eval_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "id": "KeQmLNmElkMj",
        "outputId": "11914372-4edb-4c5c-b22b-a3129b416c08"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                model_type  precision  \\\n",
              "Perceptron                                      Perceptron   0.450050   \n",
              "Passive Aggresive Classifier  Passive Aggresive Classifier   0.491443   \n",
              "Naive Bayes Classifier              Naive Bayes Classifier   0.488247   \n",
              "SGD Classifier                              SGD Classifier   0.421353   \n",
              "\n",
              "                                recall  f1_score  \n",
              "Perceptron                    0.307706  0.328190  \n",
              "Passive Aggresive Classifier  0.311450  0.361779  \n",
              "Naive Bayes Classifier        0.267576  0.332721  \n",
              "SGD Classifier                0.090096  0.143332  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3df9bb04-62b7-440a-9ccb-558a49d4d49e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model_type</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Perceptron</th>\n",
              "      <td>Perceptron</td>\n",
              "      <td>0.450050</td>\n",
              "      <td>0.307706</td>\n",
              "      <td>0.328190</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Passive Aggresive Classifier</th>\n",
              "      <td>Passive Aggresive Classifier</td>\n",
              "      <td>0.491443</td>\n",
              "      <td>0.311450</td>\n",
              "      <td>0.361779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Naive Bayes Classifier</th>\n",
              "      <td>Naive Bayes Classifier</td>\n",
              "      <td>0.488247</td>\n",
              "      <td>0.267576</td>\n",
              "      <td>0.332721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SGD Classifier</th>\n",
              "      <td>SGD Classifier</td>\n",
              "      <td>0.421353</td>\n",
              "      <td>0.090096</td>\n",
              "      <td>0.143332</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3df9bb04-62b7-440a-9ccb-558a49d4d49e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3df9bb04-62b7-440a-9ccb-558a49d4d49e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3df9bb04-62b7-440a-9ccb-558a49d4d49e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d36db121-a0c0-40be-8eeb-df1c7e69df39\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d36db121-a0c0-40be-8eeb-df1c7e69df39')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d36db121-a0c0-40be-8eeb-df1c7e69df39 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot evaluation by f1_score\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "eval_df['f1_score'].plot(kind=\"barh\")\n",
        "plt.title(\"Model evaluation by F1-score metric\", fontsize=11)\n",
        "plt.ylabel('Model type', fontsize=11)\n",
        "plt.xlabel('F1 score', fontsize=11)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "b6lx7xDyln_Q",
        "outputId": "b982cc68-f1d0-42b4-fe60-7f4d2da98116"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'F1 score')"
            ]
          },
          "metadata": {},
          "execution_count": 35
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuIAAAHGCAYAAAAxNu6CAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcUElEQVR4nO3deXwN5////+dJInskYkmCSCyJNbaqtfYgtqqiailadEG1llK101LUVi31tkX7Jt5FitpVq1o7FfsHtYU2qC0EDZL5/eGX83UkISExxON+u50bZ+Y617zmOuPkmXHNHIthGIYAAAAAPFF2ZhcAAAAAPI8I4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAJDBhg0bJovFonz58ikxMTHZ+mrVqslisahTp04Zsr0PP/xQgYGB6X5dYGCgevTokSE1ZJTMrGnJkiWaOnVqsuWdOnVSqVKlMmWbKRk2bJjc3d0zrf9atWrJYrEke9w7rlOnTlWTJk2UO3duWSwWLVq0KNPqeV5ERUVp2LBhunHjRppfY7FY9MUXX2RiVXjaOZhdAABkRdmyZdOFCxe0ceNG1apVy7r81KlT2rJlS6YGMaRsyZIl2rlzp7p162azfPDgwbp+/bpJVWWOatWqJQt4vr6+1r9/++23kqRGjRpZ/47HExUVpeHDh6tHjx5ydXVN02u2bNmigICATK4MTzOCOABkAkdHR4WGhioiIsImiC9YsEAlS5aUvb29ecXBRuHChc0uIcN5eXmpcuXKqa7fvHmz7OzsdPLkyWcmiN+8eVMuLi5ml5EhkvblQe8Rng9MTQGATNKmTRstWrRIt2/fti6bP3++2rZtm2L7jRs3qmrVqnJxcVGuXLn01ltv6dKlSzZt/v77b7388stydXVVvnz5NHbs2BT7OnPmjNq3b69cuXLJxcVFNWrU0K5dux5pP1asWKFKlSrJxcVFuXPn1nvvvWc9g3z9+nW5ubml+N/rLVu2VJUqVaztevTooaJFi8rV1VWBgYF69913FRsb+8Bt16pVS02aNLFZFhUVJYvFog0bNliXjR8/Xi+++KI8PT2VJ08eNWnSREeOHLGu79Spk+bOnasDBw5Yp2okTQ1KaWrKvn371KBBA7m5ucnT01MtW7ZUdHS0TRuLxaKxY8dq2LBh8vHxUa5cufTmm2+m+ez6jh07VLFiRTk7O6t48eJavny5dd2UKVPk6uqqq1ev2rzm0KFDslgsWrlyZZq2kRo7u0f/8T979myVLFlSLi4uypkzp1566SXt2LHDuj4xMVETJkxQ8eLF5eTkJF9fX7Vq1crmvX7YsX7y5ElZLBaFh4era9euypkzpypWrChJio+P1yeffKKAgAA5OTmpePHimj9//kPrTjqWIiIiFBQUJFdXVzVt2lSXL1/WqVOn1KBBA7m7u6tkyZI2x1aS8PBwlS5dWs7OzsqXL58GDhyohIQE67o333xTkqzTfZKmi4WHh8tisWjLli2qV6+e3Nzc9NFHH0lKeWrKihUrVK1aNbm6uipHjhyqVauWdu/enYZ3Bs8igjgAZJKmTZsqPj5ea9eulSQdPHhQe/fu1euvv56s7a5du1SvXj15eHho4cKFGjNmjH788Uc1bNjQ+sNekpo1a6YdO3Zo2rRpmjp1qn744Ydk83svX76sl156SVFRUZoyZYoWL14sNzc31alTR+fPn0/XPixatEgvv/yyQkJC9MMPP2js2LGKjIxU586dJUlubm56+eWXtWDBApvXXbt2TStWrLD+0nHjxg0lJCTos88+06pVq/Tpp5/q119/1SuvvJKuelJz5swZ9ejRQ0uXLtXMmTOVmJioqlWrWsPd4MGD1ahRIxUqVEhbtmzRli1bNHjw4BT7On36tGrUqKGLFy/qv//9r7755hv98ccfqlmzpq5du2bT9quvvtLRo0c1d+5cDRkyRPPnz9fIkSMfWu/t27fVunVrdezYUZGRkSpSpIiaN2+uffv2SZLat28vwzAUERFh87rZs2crX758atCgwQP7NwxDd+7csXlkhI0bN6pz585q1KiRVq5cqW+//VZ169bVlStXrG3ef/999evXT02aNNGPP/6or7/+Wh4eHoqLi5OU9mNdkgYMGGAdh3HjxkmSXnvtNU2fPl19+vTR8uXLFRYWpvbt22vVqlUPrX/37t2aPHmyvvjiC33zzTf67bff1LVrV7Vs2VJNmjRRZGSk8uTJo1dffdVaryRNmDBBXbp0UYMGDfTjjz+qf//++vLLLzVw4EBJUuPGjTVo0CBJ0urVq7Vlyxb98MMPNttu27at6tSpo+XLl+uNN95Isb7//e9/atq0qfLkyaP58+dr3rx5qlatmv7666+H7hueUQYAIEMNHTrUcHNzMwzDMNq2bWu0b9/eMAzDGDRokFGlShXDMAyjTJkyRseOHa2vad68uVGgQAHj1q1b1mVr1qwxJBnLli0zDMMwVq1aZUgy1q9fb21z5coVw8PDwwgICLAuGzJkiOHp6WmcO3fOuuzff/81ChQoYHz00UfWZQEBAUb37t1T3Y/ExEQjICDAaNOmjc3yVatWGRaLxdi/f79hGIaxdOlSQ5Jx5MgRa5u5c+ca9vb2xtmzZ1Ps+/bt28bvv/9uSDIOHz6cak01a9Y0GjdubPPa3bt3G5KMX375JcW+79y5Y9y4ccNwd3c3pk+fbl3esWNHo2TJksna37+8V69ehpubm3Hx4kXrskOHDhkWi8X48ssvrcskGRUrVkzWV+HChVOsK8nQoUMNScasWbNsai5YsKDx+uuvW5e1b9/epv/bt28bPj4+xieffPLA/mvWrGlISvY4ffp0srYnTpwwJBkLFy58YJ9Jxo0bZ3h7e6e6/vDhw4bFYjFGjRqVapu0HOtJdYWFhdm89ueffzYkGWvWrLFZ3rp1a+PFF198YO01a9Y03NzcjH/++ce6rE+fPoYkY9q0adZl+/btMyQZS5YsMQzDMK5evWq4u7sbAwYMsOlv2rRphouLi3HhwgXDMAxjzpw5hiSb/u9d/vnnnyerSZIxbtw4wzDu/nvLnz+/0aBBgwfuB7IWzogDQCZq06aNli5dqps3b2rBggVq06ZNiu1+++03NWvWTNmyZbMuq1+/vry8vPT7779LkrZt2yZPT0/VqVPH2sbT01OhoaE2fa1du1a1a9eWt7e39Wyovb29atasaTOF4GGOHDmiU6dO6bXXXrM5s1qzZk3Z2dlp586dkqSwsDB5eXnZnBVfsGCBateuLR8fH+uy7777TuXKlZO7u7uyZcuml156ybqdx7V161bVq1dPOXPmlIODg1xdXRUXF/dIff/222+qU6eOvL29rcuKFSumMmXKWN+LJPXq1bN5XqJECZ05cyZN22nevLn17/b29nrllVe0bds267KuXbtq+/btOnDggCRp5cqVOn/+vN56662H9p00XeTex73vRVrc+54nnakuX768Ll26pE6dOmndunXJ7hDy888/yzAM6/+YpCQtx3qSxo0b2zxfu3atvL29VadOHZv66tWrp927dyc7o36/smXLKleuXNbnwcHBkmTzbyhp2enTpyXdnU8fFxenVq1a2WwzNDRUN2/e1P79+x+4zdT25X6HDx/WmTNn0vT+IusgiANAJmrQoIGyZcumIUOG6MSJE3rttddSbHf58uUUg5KPj491ekVMTIxy586dYpt7XbhwQUuWLFG2bNlsHt999501XKTFhQsXJN0NjPf24+rqqoSEBGtfjo6OatGihTWIX7x4UevWrbOZC//DDz+oQ4cOqlixor7//ntt3brV+l/3//77b5prSkl0dLTq16+vhIQETZ8+XZs2bdKOHTuUJ0+eR+o7Le9FEi8vL5vnjo6Oio+Pf+g2smXLphw5ciTrPyYmxvq8Ro0aKlq0qGbNmiXp7rSUGjVqpOniUk9PT1WoUMHmcW/wfZiTJ0/avOdJ26xTp46+++47HThwQA0aNFCuXLnUoUMH67hcvHhRDg4OypMnT6p9p2d8Uzq2L126lOzY7tKli+7cuWMzfilJ6f26f3nSsqRjJ+nfQfny5W22GRQUJElp/jf1sF+ELl68KEnKmzdvmvpD1sBdUwAgE2XLlk0tWrTQhAkTVLdu3VR/GHt7e6c4f/vcuXPWM7N+fn76559/Umxzf19hYWEpzlV2cnJKc+1J2/3qq69UqVKlZOvvDQxt2rTRrFmztHfvXm3ZskX29vZ69dVXresXLlyosmXLavr06dZlv/7660NrcHZ21q1bt2yWXb582eb56tWrFRcXp8jISGugunPnTrJQl1YPei+SzpY+rtu3b+vy5cs2YfzcuXPy8/OzadelSxeNHTtWvXv31ooVKzR79uwM2f7D5M2b1+Z/T+49btq3b6/27dvrwoULWrp0qXr16qVs2bJp1qxZypkzp+7cuaPz58+nGsbTcqwnsVgsyV6bO3fuVC9WfdAvAI8qqabIyEj5+/snW1+wYME09XP/vtwvZ86cku5ekI3nB0EcADJZly5ddP78eXXt2jXVNi+99JKWLFmi8ePHy8Hh7kfzunXrdOXKFesUjooVKyo2NlY///yzdXpKbGysfvrpJ5sAExoaqv/+978qXry43NzcHrnuYsWKKX/+/Dp+/Li6d+/+wLa1atWSr6+vIiIitGXLFjVs2FCenp7W9Tdv3rSeaUwyb968h9aQP39+rVu3ToZhWINM0sWv9/ZtsVhszvh+//33yS5QdHR0TNMZ8pdeekn/+c9/bILy4cOHtXfv3gydNvDDDz9Y+0tISNCSJUuS/cLTsWNHDRw4UO3atZOrq6tatmyZYdt/EEdHR1WoUOGBbXLlyqXOnTtr5cqVOnTokKS7Z8wtFovmzJmj/v37p/i6tBzrqQkNDdXYsWPl6Oio0qVLP8KepV+VKlXk6uqqM2fO2Ewnut/9Z9LTq2jRosqfP7/mzJmT6v+cIeshiANAJqtYsaKWLFnywDYDBw5U1apV1aRJE73//vs6d+6cPv74Y1WsWFGNGjWSdHcudvny5dWuXTuNGTNGXl5eGj16tLJnz27TV+/evTVv3jzVrFlTH3zwgQoUKKB//vlH27ZtU968edWrV6801W2xWDRhwgS1bdtW169fV+PGjeXm5qZTp05pxYoVGjVqlPUMsb29vV577TWFh4fr/Pnzye6iUq9ePXXv3l0jR45UlSpVtHLlSq1fv/6hNbRs2VKzZs3S+++/r1deeUWbN29OdpeYpF9K3nzzTb3zzjs6cOCAxo8fn2waQvHixTV79mzr7ety5cqV4jeS9urVS3PmzFH9+vU1cOBA/fvvvxo0aJAKFCiQYd+G6ujoqE8//VT//vuvChYsqKlTp+r06dPJjpPcuXOrWbNmWrhwod55550Mu4/2zp07dfLkSev/sGzdutW6vZo1a6b6uqFDh+rixYuqVauW8uTJo3379mn16tXq3bu3pLvzq999910NGjRIly5dUt26dXXjxg2tWLFCw4YNs97272HHemrq1aunpk2bKiwsTP369VPp0qV1/fp1HThwQH/++admzpyZIeNzLy8vL40YMUL9+vXTmTNnVKtWLdnb2+v48eNaunSpFi9eLFdXVxUvXlyS9PXXX+uVV16Rq6urQkJC0rydpFsZtmnTRi1atFCHDh3k5OSkLVu26MUXX0x2G09kEWZfLQoAWc29d01Jzf13TTEMw9iwYYNRpUoVw8nJyfD29jY6depkc+cOwzCM06dPG40bNzacnZ0NPz8/Y9SoUcYHH3xgc9cUwzCMmJgYo3Pnzoafn5/h6Oho5M+f32jZsqWxadMma5uH3TUlydq1a613nHBzczNKlixp9OnTx7hy5YpNuy1bthiSDHd3d+PGjRs26+7cuWP06dPHyJ07t+Hh4WG0bNnS2Lp1a7I7dqRU09ixYw1/f3/Dzc3NaNWqlfHTTz8lu2vKt99+axQqVMhwdnY2KleubGzfvj1ZX7Gxscbrr79u5MyZ05BkHf+U7qayZ88eo169eoarq6vh4eFhvPrqq8bJkydt2uieO14kmThxovGwH61Jx8fWrVuNF154wXB0dDSKFi1qLF26NMX28+fPNyQZ27Zte2C/SVK608z9OnbsmOKdVWrWrPnA1/34449G3bp1jdy5cxtOTk5G4cKFjaFDhxq3b9+2tklISDDGjh1rBAUFGdmyZTN8fX2N1q1bG7GxsdY2DzvWH3Q3l/j4eGP48OFGUFCQ4ejoaOTOnduoXbu28e2336Z7XFK700lK721ERITx4osvGi4uLkb27NmNcuXKGYMHD7bZ92HDhhn58+c37OzsrP8mU9tGattZtmyZUalSJcPZ2dnw8vIy6tSpY+zevfuB+4Znl8UwDOOJp38AAJAmHTp00O7du633GAeQdTA1BQCAp9C+ffsUFRWlBQsWaOrUqWaXAyATcEYcAICnUGBgoP755x+1bdtW06dPf6yvpQfwdCKIAwAAACbg12sAAADABARxAAAAwAQEcQAAAMAE3DUFeIolJibq77//loeHx0O/HhkAADwdDMPQtWvXlDdv3gdeaE0QB55if//9t/z9/c0uAwAAPILTp08rf/78qa4niANPMQ8PD0l3/yHf/zXmAADg6XT16lX5+/tbf46nhiAOPMWSpqNkz56dIA4AwDPmYdNKuVgTAAAAMAFBHAAAADABQRwAAAAwAUEcAAAAMAFBHAAAADABQRwAAAAwAUEcAAAAMAFBHAAAADABQRwAAAAwAUEcAAAAMAFBHAAAADABQRwAAAAwAUEcAAAAMAFBHAAAADABQRwAAAAwAUEcAAAAMAFBHAAAADABQRwAAAAwAUEcAAAAMAFBHAAAADCBg9kFAHi4UkPXyM7J1ewygEx18vPGZpcAAE8UZ8QBAAAAExDEAQAAABMQxAEAAAATEMQBAAAAExDEAQAAABMQxAEAAAATEMQBAAAAExDEAQAAABMQxAEAAAATEMQBAAAAExDEAQAAABMQxAEAAAATEMQBAAAAExDEAQAAABMQxAEAAAATEMQBAAAAExDEAQAAABMQxPFMCA8Pl5eX1xPZVqdOnfTKK69YnxuGobffflve3t6yWCyKiopSrVq19OGHHz6RegAAQNZEEH9O/fPPP3rvvfdUoEABOTk5ydfXVw0aNNCmTZts2u3evVutW7eWn5+fnJycFBAQoCZNmujHH3+UYRiSpJMnT8pisVgfHh4eKlmypLp3766jR4+mqZ5ffvlFjRo1Us6cOeXq6qoSJUqoT58++uuvvzJ83x9m8uTJCg8Ptz5fvXq1wsPDtXz5csXExKhUqVKKjIzUyJEjn3htAAAg6yCIP6datGih3bt3a+7cuTpy5IiWLVumWrVq6eLFi9Y2S5cuVeXKlRUXF6e5c+fq0KFDWr16tZo3b65BgwYpNjbWps+ffvpJMTEx2rNnj0aNGqVDhw6pTJkyWr9+/QNrmT59ukJDQ+Xr66vFixfr4MGD+uabbxQbG6vx48dnyv4/iKenp83Z92PHjsnPz09Vq1aVr6+vHBwc5O3tLQ8Pj0feRkJCghITEzOgWgAA8KwiiD+Hrly5ot9++01jxoxR7dq1FRAQoIoVK2rAgAF6+eWXJUnXr19X586d1bhxY61YsUL169dXoUKFVLx4cXXu3Fl79uyRp6enTb85c+aUr6+vChUqpGbNmumnn35SpUqV1LlzZyUkJKRYy5kzZ9SzZ0/17NlTs2fPVq1atRQYGKgaNWpo5syZGjJkSIqvO3bsmJo1ayYfHx+5u7vrxRdf1E8//WTTZurUqQoKCpKzs7N8fHzUsmVL67pFixYpJCRELi4uypkzp0JDQ3X9+nVJtlNTOnXqpPfff1/R0dGyWCwKDAyUpGRTU+Lj49W3b1/ly5dPbm5uqlSpkjZs2GBdnzS1ZtmyZSpRooScnJwUHR390PcKAABkXQTx55C7u7vc3d21ZMkSxcfHp9hm7dq1unjxovr165dqPxaL5YHbsbOz0wcffKBTp05p165dKbZZuHChbt26lep2UpsXHhcXp0aNGmn9+vXavXu3wsLC1LRpU2u43blzp3r27KkRI0bo8OHDWr16tWrUqCFJiomJUZs2bfTWW2/p0KFD2rBhg1599VXrVJt7TZ48WSNGjFD+/PkVExOjHTt2pFhPjx49tGXLFi1YsEB79+5Vq1atFBYWZjM158aNGxozZoxmzpypAwcOKE+ePMn6iY+P19WrV20eAAAgayKIP4ccHBwUHh6uuXPnysvLS9WqVdMnn3yivXv3WtscOXJEklS0aFHrsh07dlhDvLu7u5YvX/7QbRUrVkzS3XnkKTl69KiyZ88uPz+/dO1DmTJl9M4776hUqVIKCgrSyJEjVbhwYS1btkySFB0dLTc3NzVp0kQBAQEqV66cevbsKeluEL9z545effVVBQYGKiQkRN26dZO7u3uy7Xh6esrDw0P29vby9fVV7ty5k7WJjo7WnDlztHDhQlWvXl2FCxdW37599dJLL2nOnDnWdrdv39bUqVNVtWpVFS1aVK6ursn6Gj16tDw9Pa0Pf3//dI0LAAB4dhDEn1MtWrTQ33//rWXLliksLEwbNmxQ+fLlbS5SvF/p0qUVFRWlqKgoXb9+XXfu3HnodpLOMqd29twwjIeeWU9JXFyc+vbtq+LFi8vLy0vu7u46dOiQ9Yx4vXr1FBAQoEKFCumNN97QvHnzdOPGDUl3Q3zdunUVEhKiVq1aacaMGbp8+XK6a0iyb98+JSQkKDg42OYXlV9//VXHjh2ztnN0dFTp0qUf2NeAAQMUGxtrfZw+ffqR6wIAAE83gvhzzNnZWfXq1dPgwYO1efNmderUSUOHDpUkBQUFSZIOHz5sbe/k5KQiRYqoSJEiad7GoUOHJEkFCxZMcX1wcLBiY2MVExOTrtr79u2rH374QaNGjdJvv/2mqKgohYSE6NatW5IkDw8P/fHHH4qIiJCfn5+GDBmiMmXK6MqVK7K3t9e6deu0atUqlShRQlOmTFHRokV14sSJdNWQJC4uTvb29tq1a5f1F5WoqCgdOnRIkydPtrZzcXF56C8dTk5Oyp49u80DAABkTQRxWJUoUcJ6wWL9+vXl7e2tMWPGPHJ/iYmJ+vLLL1WwYEGVK1cuxTYtW7aUo6Ojxo4dm+L6K1eupLh806ZN6tSpk5o3b66QkBD5+vomm/7i4OCg0NBQjR07Vnv37tXJkyf1888/S7p7hr5atWoaPny4du/eLUdHR/3www+PtJ/lypVTQkKCzp8/b/1FJenh6+v7SH0CAICsz8HsAvDkXbx4Ua1atdJbb72l0qVLy8PDQzt37tTYsWPVrFkzSXcv6Jw5c6Zat26txo0bq2fPngoKClJcXJxWr14tSbK3t0/W79mzZ3Xjxg3t379fkyZN0vbt27VixYpkbZP4+/tr4sSJ6tGjh65evaoOHTooMDBQZ86c0bfffit3d/cUb2EYFBSkyMhINW3aVBaLRYMHD7a5HeDy5ct1/Phx1ahRQzly5NDKlSuVmJiookWLatu2bVq/fr3q16+vPHnyaNu2bfrnn39UvHjxRxrP4OBgtWvXTh06dND48eNVrlw5/fPPP1q/fr1Kly6txo0bP1K/AAAgayOIP4fc3d1VqVIlTZw4UceOHdPt27fl7++vrl276pNPPrG2a968uTZv3qwxY8aoQ4cOunTpkjw9PVWhQgUtWLBATZo0sek3NDRUkuTq6qqAgADVrl1b//nPfx46laVbt24KDg7WF198oebNm+vmzZsKDAxUkyZN1Lt37xRfM2HCBL311luqWrWqcuXKpf79+9vcYcTLy0uRkZEaNmyY/v33XwUFBSkiIkIlS5bUoUOHtHHjRk2aNElXr15VQECAxo8fr4YNGz7qkGrOnDn69NNPrV9ClCtXLlWuXDnZGAEAACSxGCndsw3AU+Hq1at3757y4feyc0p+lxUgKzn5Of97BCBrSPr5HRsb+8DrvZgjDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJjAwewCADzc/uENlD17drPLAAAAGYgz4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJHMwuAMDDlRq6RnZOrmaXAQBAup38vLHZJTy1OCMOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAg/gTUqlVLH374odllPNOGDRumsmXLPpFt3f9+3bhxQy1atFD27NllsVh05coVBQYGatKkSU+kHgAAkDURxFPRqVMnWSwWff755zbLlyxZIovFkq6+IiMjNXLkyIwsL5mkepMeOXPmVFhYmPbu3Zup280oixcvVq1ateTp6Sl3d3eVLl1aI0aM0KVLl554Lfe/X3PnztVvv/2mzZs3KyYmRp6entqxY4fefvvtJ14bAADIOgjiD+Ds7KwxY8bo8uXLj9WPt7e3PDw8Mqiq1IWFhSkmJkYxMTFav369HBwc1KRJk0zf7uMaOHCgWrdurRdffFGrVq3S/v37NX78eO3Zs0fffffdE6/n/vfr2LFjKl68uEqVKiVfX19ZLBblzp1brq6uj7yNW7duZUSpAADgGUYQf4DQ0FD5+vpq9OjRqba5ePGi2rRpo3z58snV1VUhISGKiIiwaXPvVIdPPvlElSpVStZPmTJlNGLECOvzmTNnqnjx4nJ2dlaxYsU0derUh9br5OQkX19f+fr6qmzZsvr44491+vRp/fPPP9Y2/fv3V3BwsFxdXVWoUCENHjxYt2/fliSdPHlSdnZ22rlzp02/kyZNUkBAgBITEyVJ+/fvV8OGDeXu7i4fHx+98cYbunDhgrX9okWLFBISIhcXF+XMmVOhoaG6fv16ijVv375do0aN0vjx4zVu3DhVrVpVgYGBqlevnhYvXqyOHTum+LodO3aoXr16ypUrlzw9PVWzZk398ccf1vWGYWjYsGEqUKCAnJyclDdvXvXs2dO6furUqQoKCpKzs7N8fHzUsmVL67p7369atWpp/Pjx2rhxoywWi2rVqiVJyaamXLlyRV26dFHu3LmVPXt21alTR3v27LGuT5paM3PmTBUsWFDOzs4p7hcAAHh+EMQfwN7eXqNGjdKUKVN05syZFNv8+++/euGFF7RixQrt379fb7/9tt544w1t3749xfbt2rXT9u3bdezYMeuyAwcOaO/evWrbtq0kad68eRoyZIg+++wzHTp0SKNGjdLgwYM1d+7cNNceFxen//73vypSpIhy5sxpXe7h4aHw8HAdPHhQkydP1owZMzRx4kRJd8NlaGio5syZY9PXnDlz1KlTJ9nZ2enKlSuqU6eOypUrp507d2r16tU6d+6cXnvtNUlSTEyM2rRpo7feekuHDh3Shg0b9Oqrr8owjBTrnDdvntzd3dWtW7cU13t5eaW4/Nq1a+rYsaN+//13bd26VUFBQWrUqJGuXbsm6e5Ul4kTJ2r69Ok6evSolixZopCQEEnSzp071bNnT40YMUKHDx/W6tWrVaNGjRS3ExkZqa5du6pKlSqKiYlRZGRkiu1atWql8+fPa9WqVdq1a5fKly+vunXr2kyt+fPPP7V48WJFRkYqKioqxX4AAMDzw8HsAp52zZs3V9myZTV06FDNmjUr2fp8+fKpb9++1ufvv/++1qxZo++//14VK1ZM1r5kyZIqU6aM5s+fr8GDB0u6G0YrVaqkIkWKSJKGDh2q8ePH69VXX5UkFSxYUAcPHtT06dNTPUMsScuXL5e7u7sk6fr16/Lz89Py5ctlZ/f/ft8aNGiQ9e+BgYHq27evFixYoH79+kmSunTponfffVcTJkyQk5OT/vjjD+3bt09Lly6VJH311VcqV66cRo0aZe1n9uzZ8vf315EjRxQXF6c7d+7o1VdfVUBAgCRZA3BKjh49qkKFCilbtmyptklJnTp1bJ7/5z//kZeXl3799Vc1adJE0dHR8vX1VWhoqLJly6YCBQpY34/o6Gi5ubmpSZMm8vDwUEBAgMqVK5fidry9veXq6ipHR0f5+vqm2Ob333/X9u3bdf78eTk5OUmSvvjiCy1ZskSLFi2yziW/deuWvv32W+XOnTvV/YqPj1d8fLz1+dWrV9M+KAAA4JnCGfE0GDNmjObOnatDhw4lW5eQkKCRI0cqJCRE3t7ecnd315o1axQdHZ1qf+3atdP8+fMl3Z1CERERoXbt2km6G6CPHTumzp07y93d3fr49NNPbc6ip6R27dqKiopSVFSUtm/frgYNGqhhw4Y6deqUtc3//vc/VatWTb6+vnJ3d9egQYNsan3llVdkb2+vH374QZIUHh6u2rVrKzAwUJK0Z88e/fLLLza1FStWTNLdudRlypRR3bp1FRISolatWmnGjBkPnGOf2pnyhzl37py6du2qoKAgeXp6Knv27IqLi7PuS6tWrXTz5k0VKlRIXbt21Q8//KA7d+5IkurVq6eAgAAVKlRIb7zxhubNm6cbN248Uh3S3TGJi4tTzpw5bcblxIkTNu9ZQEDAA0O4JI0ePVqenp7Wh7+//yPXBQAAnm4E8TSoUaOGGjRooAEDBiRbN27cOE2ePFn9+/fXL7/8oqioKDVo0OCBF+O1adNGhw8f1h9//KHNmzfr9OnTat26taS7U0okacaMGdZQHRUVpf3792vr1q0PrNPNzU1FihRRkSJF9OKLL2rmzJm6fv26ZsyYIUnasmWL2rVrp0aNGmn58uXavXu3Bg4caFOro6OjOnTooDlz5ujWrVuaP3++3nrrLev6uLg4NW3a1Ka2qKgoHT16VDVq1JC9vb3WrVunVatWqUSJEpoyZYqKFi2qEydOpFhzcHCwjh8/bp2nnlYdO3ZUVFSUJk+erM2bNysqKko5c+a07ou/v78OHz6sqVOnysXFRd26dVONGjV0+/ZteXh46I8//lBERIT8/Pw0ZMgQlSlTRleuXElXDfeOiZ+fX7IxOXz4sD766CNrOzc3t4f2NWDAAMXGxlofp0+ffqSaAADA04+pKWn0+eefq2zZsipatKjN8k2bNqlZs2Zq3769JCkxMVFHjhxRiRIlUu0rf/78qlmzpubNm6ebN2+qXr16ypMnjyTJx8dHefPm1fHjx61nyR+VxWKRnZ2dbt68KUnavHmzAgICNHDgQGube8+WJ+nSpYtKlSqlqVOnWqeZJClfvrwWL16swMBAOTikfPhYLBZVq1ZN1apV05AhQxQQEKAffvhBvXv3Tta2bdu2+vLLLzV16lR98MEHydZfuXIlxXnimzZt0tSpU9WoUSNJ0unTp20uGJUkFxcXNW3aVE2bNlX37t1VrFgx7du3T+XLl5eDg4NCQ0MVGhqqoUOHysvLSz///LPNvqZV+fLldfbsWTk4OFj/5+BROTk5Wae3AACArI0gnkYhISFq166dvvzyS5vlQUFBWrRokTZv3qwcOXJowoQJOnfu3AODuHR3esrQoUN169Yt68WSSYYPH66ePXvK09NTYWFhio+P186dO3X58uUUw2yS+Ph4nT17VpJ0+fJlffXVV9Yz2Em1RkdHa8GCBXrxxRe1YsUK6xSUexUvXlyVK1dW//799dZbb8nFxcW6rnv37poxY4batGmjfv36ydvbW3/++acWLFigmTNnaufOnVq/fr3q16+vPHnyaNu2bfrnn39UvHjxFGuuVKmS+vXrpz59+uivv/5S8+bNlTdvXv3555/65ptv9NJLL6UY0IOCgvTdd9+pQoUKunr1qj766CObOsPDw5WQkKBKlSrJ1dVV//3vf+Xi4qKAgAAtX75cx48fV40aNZQjRw6tXLlSiYmJyX7JSqvQ0FBVqVJFr7zyisaOHavg4GD9/fffWrFihZo3b64KFSo8Ur8AACBrY2pKOowYMcJ6C78kgwYNUvny5dWgQQPVqlVLvr6+euWVVx7aV8uWLXXx4kXduHEjWfsuXbpo5syZmjNnjkJCQlSzZk2Fh4erYMGCD+xz9erV8vPzk5+fnypVqqQdO3Zo4cKF1lvuvfzyy+rVq5d69OihsmXLavPmzdYLRu/XuXNn3bp1y2ZaiiTlzZtXmzZtUkJCgurXr6+QkBB9+OGH8vLykp2dnbJnz66NGzeqUaNGCg4O1qBBgzR+/Hg1bNgw1brHjBmj+fPna9u2bWrQoIFKliyp3r17q3Tp0qlenDpr1ixdvnxZ5cuX1xtvvKGePXta/1dBunu3lRkzZqhatWoqXbq0fvrpJ/3444/KmTOnvLy8FBkZqTp16qh48eL65ptvFBERoZIlSz5wfFNjsVi0cuVK1ahRQ2+++aaCg4P1+uuv69SpU/Lx8XmkPgEAQNZnMR71ajndnQ5w+vRplSlTJk3zX/HsGDlypBYuXPjMfDNnVnX16tW7F21++L3snB79C4QAADDLyc8bm13CE5f08zs2NlbZs2dPtd0jnRH/z3/+o3z58ikwMFDVq1fX4cOHJd291d/kyZMfrWI8FeLi4rR//3599dVXev/9980uBwAAIMtKdxCfNGmS3n//fXXo0EFr1qyxuf1crVq1tHDhwgwtEE9Wjx499MILL6hWrVrJpqUAAAAg46T7Ys0pU6Zo8ODBGjRokBISEmzWFS1a1Hp2HM+m8PBwhYeHm10GAABAlpfuM+J//fWXqlatmuK6bNmyWe+DDQAAACB16Q7iAQEB2r59e4rrtm3bpuDg4McuCgAAAMjq0h3Eu3btqk8//VSzZs3S1atXJUm3b9/WihUrNG7cOL3zzjsZXiQAAACQ1aR7jnjfvn0VHR2tt99+2xq6q1WrJknq1q2bunXrlrEVAgAAAFnQI32z5pdffqkPPvhAP/30ky5evChvb2/VrVtXQUFBGV0fAAAAkCU98lfcFy5cWIULF87IWgAAAIDnxiMF8du3bys8PFzbtm1TTEyM/Pz8VLlyZXXs2FHZsmXL6BoBAACALCfdF2seOXJERYsW1Xvvvafdu3fLMAzt3r1b7777roKDg7mPOAAAAJAG6T4j/s4778jR0VGHDx+2mZry559/qmnTpnrvvff0888/Z2iRAAAAQFaT7jPi27Zt02effZZsfniRIkU0YsQIbd26NcOKAwAAALKqdAfxvHnzymKxpLjOYrHI19f3sYsCAAAAsrp0B/GhQ4dq8ODBOn78uM3y48ePa+jQoRo6dGiGFQcAAABkVemeI/7999/rypUrKlq0qEqVKqU8efLo/Pnz2r9/v3x8fLR48WItXrxY0t0z5EuXLs3wogEAAIBnXbqDeFxcnIKDgxUcHCxJunXrlry8vPTSSy9Jkq5du5axFQIAAABZkMUwDMPsIgCk7OrVq/L09FRsbKyyZ89udjkAACAN0vrzO91zxFevXi2yOwAAAPB40h3EGzVqpICAAA0ZMkQnTpzIjJoAAACALC/dQfzAgQNq3bq1ZsyYoaCgINWpU0fz589XfHx8ZtQHAAAAZEnpDuLFixfXuHHjdObMGUVGRip79uzq1KmTfH191b17d+3atSsz6gQAAACylHQH8ST29vZ6+eWXtWTJEp04cUJlypTRtGnTVLFiRZUpU0Zz5szJyDoBAACALOWRg7gkHTp0SH379tULL7ygrVu3qlWrVpo3b57KlCmjd999V++++25G1QkAAABkKem+fWFcXJwiIiI0e/Zsbd++XcHBwerSpYs6duyoXLlyWdt999136tatG/cVBx4Dty8EAODZk9af3+n+Qh8fHx9JUosWLTR27FhVr149xXblypWzCeYAAAAA/p90B/ExY8aoffv28vLyemC7UqVKcXtDAAAAIBXpniN+6dIl3bhxI8V1MTExGjFixGMXBQAAAGR16Q7iw4cP15kzZ1Jc9/fff2v48OGPXRQAAACQ1aU7iBuGIYvFkuK6mJiYh05ZAQAAAJDGOeIRERGKiIiQJFksFvXp0ydZ4P7333+1c+dOVatWLcOLBAAAALKaNAXxW7duWW9DaBiGrl+/Lnt7e5s2jo6O6tChg/r165fxVQIAAABZTLrvI167dm1NmzZNxYoVy6yaAPz/uI84AADPnky7j/gvv/zyWIUBAAAAeMyvuAcAAADwaAjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAnSdNeUnj17prlDi8WiyZMnP3JBAAAAwPMgTfcRL1iwYNo7tFh0/PjxxyoKwF3cRxwAgGdPht5H/MSJExlWGAAAAADmiAMAAACmeKQgfuHCBX388ceqW7eugoODdeDAAUnS5MmTtXXr1gwtEAAAAMiK0h3E//jjDwUFBWnBggXKnz+/jh07pvj4eEnSX3/9pYkTJ2Z4kQAAAEBWk+4g3qtXL1WpUkVHjx7VrFmzdO+1npUqVeKMOAAAAJAGabpY8147duxQZGSksmXLpoSEBJt1uXPn1vnz5zOsOAAAACCrSvcZcTc3N129ejXFddHR0cqZM+djFwUAAABkdekO4g0aNNCnn36qixcvWpdZLBbdvHlTkydPVqNGjTK0QAAAACArStMX+tzrr7/+UrVq1XT16lXVrl1bS5YsUVhYmA4ePCiLxaKtW7cqT548mVUv8FzhC30AAHj2pPXnd7rPiOfLl09RUVF6//33FRMTo8KFC+vixYtq166ddu7cSQgHAAAA0iDdZ8QBPDmcEQcA4NmTaWfEAQAAADy+NN2+sGDBgrJYLGnu9Pjx449cEAAAAPA8SFMQb9asmU0QX7Roka5evarQ0FD5+Pjo3Llz+umnn+Tp6amWLVtmWrEAAABAVpGmID5p0iTr38eNGyd/f3+tXr3aZs5LbGysGjZsKB8fnwwvEgAAAMhq0j1H/Msvv9SAAQOSTTz39PTUxx9/rClTpmRYcQAAAEBWle4gfunSJcXGxqa4LjY2VpcvX37sogAAAICsLt1BvG7duurfv79+/fVXm+UbNmzQxx9/rLp162ZYcQAAAEBWle4gPn36dOXNm1d16tSRt7e3ihYtKm9vb9WtW1d+fn765ptvMqNOAAAAIEtJ08Wa9/Lz89OOHTu0evVqbd++XTExMfLz81PFihUVFhaWGTUCAAAAWQ7frAk8xfhmTQAAnj1p/fmd7jPikmQYhlauXKnff/9dly5dkre3t6pXr66GDRum64t/AAAAgOdVus+IX758WY0aNdK2bdvk5eVl/UKfK1euqHLlylq5cqW8vLwyqVzg+ZL0G7X/h9/LzsnV7HIAAMgyTn7eONP6TusZ8XRfrNm3b18dO3ZMa9as0aVLl3To0CFdunRJa9as0bFjx9S3b9/HKhwAAAB4HqQ7iC9btkxjxoxRvXr1bJbXq1dPo0eP1tKlSzOsOAAAACCrSncQv379eqpfY+/r66vr168/dlEAAABAVpfuIF6uXDl99dVXSkhIsFmemJioKVOmqHz58hlWHAAAAJBVpfuuKaNHj1b9+vVVpEgRNWvWTD4+Pjp//ryWLFmis2fPau3atZlRJwAAAJClpDuI16hRQ5s2bdJnn32m+fPn6/Lly/L29tZLL72kgQMHckYcAAAASINHuo/4Cy+8oMjIyIyuBQAAAHhupHuOOAAAAIDHl6Yz4m+99VaaO7RYLJo1a9YjFwQAAAA8D9IUxMPDw+Xh4aHChQvrYV/EyVfcAwAAAA+XpiBepUoVbd26VQkJCWrbtq1ef/11BQQEZHZtAAAAQJaVpjnimzZt0okTJ9SuXTtFRESoUKFCeumllzR16lRduHAhs2sEAAAAspw0X6xZoEAB9evXT1FRUdq3b59q166tSZMmKW/evGrYsKFWrVqVmXUCAAAAWcoj3TWlRIkSGjlypPbu3asPP/xQ69at04wZMzK6NgAAACDLSvd9xBMSErR27VotWLBAS5culYODgzp37qwuXbpkRn0AAABAlpTmIL5x40ZFRERo4cKFio+PV7NmzTRv3jw1aNBADg6P9L1AAAAAwHMrTQna399fFy5cUMOGDTVt2jQ1bdpUzs7OmV0bAAAAkGWlaY74X3/9pcTERK1bt06dO3dWnjx5lD179hQfnp6emV1zhjh58qQsFouioqLMLuWpVqtWLX344YdmlyHp7j3qlyxZkunb2bBhgywWi65cuWJdtmTJEhUpUkT29vb68MMPFR4eLi8vr0yvBQAAZF1pOiM+dOjQTCugU6dOmjt3riQpW7ZsKlCggDp06KBPPvkkU6e8+Pv7KyYmRrly5cq0bdzrnXfe0cyZM7VgwQK1atXqiWwzI0RGRipbtmyZvp2zZ8/qs88+04oVK/TXX38pT548Klu2rD788EPVrVs307d/r6pVqyomJsbml8p33nlHb775pnr27CkPDw85ODioUaNGT7QuAACQtZgexCUpLCxMc+bMUXx8vFauXKnu3bsrW7ZsGjBgQKZt097eXr6+vpnW/71u3LihBQsWqF+/fpo9e/YTC+K3bt2So6PjY/Xh7e2dQdWk7uTJk6pWrZq8vLw0btw4hYSE6Pbt21qzZo26d++u//u//8v0Gu7l6Ohoc2zExcXp/PnzatCggfLmzWtd7uLi8ljbuX379hP5JQcAADydHun2hRnNyclJvr6+CggI0HvvvafQ0FAtW7ZMkjRhwgSFhITIzc1N/v7+6tatm+Li4qyvPXXqlJo2baocOXLIzc1NJUuW1MqVKyVJly9fVrt27ZQ7d265uLgoKChIc+bMkWQ7NSUxMVH58+fXtGnTbOravXu37OzsdOrUKUnSlStX1KVLF+XOnVvZs2dXnTp1tGfPnofu38KFC1WiRAl9/PHH2rhxo06fPm2z/s6dO+rZs6e8vLyUM2dO9e/fXx07dtQrr7xibXPt2jW1a9dObm5u8vPz08SJE5NNGwkMDNTIkSPVoUMHZc+eXW+//bYk6ffff1f16tXl4uIif39/9ezZU9evX7e+burUqQoKCpKzs7N8fHzUsmVL67p7t/HJJ5+oUqVKyfavTJkyGjFihPX5zJkzVbx4cTk7O6tYsWKaOnXqA8enW7duslgs2r59u1q0aKHg4GCVLFlSvXv31tatW1N9Xf/+/RUcHCxXV1cVKlRIgwcP1u3bt63r9+zZo9q1a8vDw0PZs2fXCy+8oJ07d0p68HFz79SUDRs2yMPDQ5JUp04dWSwWbdiwIcWpKUuXLlX58uXl7OysQoUKafjw4bpz5451vcVi0bRp0/Tyyy/Lzc1Nn3322QPHBQAAZG1PRRC/n4uLi27duiVJsrOz05dffqkDBw5o7ty5+vnnn9WvXz9r2+7duys+Pl4bN27Uvn37NGbMGLm7u0uSBg8erIMHD2rVqlU6dOiQpk2bluJUFDs7O7Vp00bz58+3WT5v3jxVq1ZNAQEBkqRWrVrp/PnzWrVqlXbt2qXy5curbt26unTp0gP3Z9asWWrfvr08PT3VsGFDhYeH26wfM2aM5s2bpzlz5mjTpk26evVqsrnQvXv31qZNm7Rs2TKtW7dOv/32m/74449k2/riiy9UpkwZ7d69W4MHD9axY8cUFhamFi1aaO/evfrf//6n33//XT169JAk7dy5Uz179tSIESN0+PBhrV69WjVq1EhxP9q1a6ft27fr2LFj1mUHDhzQ3r171bZtW+uYDRkyRJ999pkOHTqkUaNGafDgwdbpR/e7dOmSVq9ere7du8vNzS3Z+gfNw/bw8FB4eLgOHjyoyZMna8aMGZo4caJNvfnz59eOHTu0a9cuffzxx9Yz0A86bu5VtWpVHT58WJK0ePFixcTEqGrVqsna/fbbb+rQoYM++OADHTx4UNOnT1d4eHiysD1s2DA1b95c+/bt01tvvZWsn/j4eF29etXmAQAAsqan6r6DhmFo/fr1WrNmjd5//31JSnbG99NPP9W7775rPcsaHR2tFi1aKCQkRJJUqFAha/vo6GiVK1dOFSpUsL4+Ne3atdP48eMVHR2tAgUKKDExUQsWLNCgQYMk3T2rvH37dp0/f15OTk6S7obeJUuWaNGiRdazz/c7evSotm7dqsjISElS+/bt1bt3bw0aNEgWi0WSNGXKFA0YMEDNmzeXJH311VfWs7PS3bPhc+fO1fz5863zpefMmWMzTSJJnTp11KdPH+vzLl26qF27dtZxDAoK0pdffqmaNWtq2rRpio6Olpubm5o0aSIPDw8FBASoXLlyKe5LyZIlVaZMGc2fP1+DBw+WdDd4V6pUSUWKFJF0dxrT+PHj9eqrr0qSChYsaA2mHTt2TNbnn3/+KcMwVKxYsRS3+SBJ7410973t27evdQqQdPf9/+ijj6x9BwUFWds/6Li5l6Ojo/LkySPp7jSd1KYzDR8+XB9//LF1HwsVKqSRI0eqX79+NlO72rZtqzfffDPVfRo9erSGDx/+0H0HAADPvqfijPjy5cvl7u4uZ2dnNWzYUK1bt9awYcMkST/99JPq1q2rfPnyycPDQ2+88YYuXryoGzduSJJ69uypTz/9VNWqVdPQoUO1d+9ea7/vvfeeFixYoLJly6pfv37avHlzqjWULVtWxYsXt54V//XXX3X+/HnrfO49e/YoLi5OOXPmlLu7u/Vx4sQJmzPE95s9e7YaNGhgPRPfqFEjxcbG6ueff5YkxcbG6ty5c6pYsaL1Nfb29nrhhResz48fP67bt2/btPH09FTRokWTbS/pl44ke/bsUXh4uE3NDRo0UGJiok6cOKF69eopICBAhQoV0htvvKF58+ZZxzYl7dq1s46RYRiKiIhQu3btJEnXr1/XsWPH1LlzZ5vtffrpp6mOkWEYqW7rYf73v/+pWrVq8vX1lbu7uwYNGqTo6Gjr+t69e6tLly4KDQ3V559/blPDg46bR7Fnzx6NGDHCZr+7du2qmJgYm/G8//2534ABAxQbG2t93D+NCQAAZB1PRRCvXbu2oqKidPToUd28eVNz586Vm5ubTp48qSZNmqh06dJavHixdu3apa+//lqSrFNXunTpouPHj+uNN97Qvn37VKFCBU2ZMkWS1LBhQ506dUq9evXS33//rbp166pv376p1nFvyJw/f77CwsKUM2dOSXcv2PPz81NUVJTN4/Dhw/roo49S7C8hIUFz587VihUr5ODgIAcHB7m6uurSpUuaPXt2ho3fve6f3hEXF6d33nnHpuY9e/bo6NGjKly4sDw8PPTHH38oIiJCfn5+GjJkiMqUKWNz6757tWnTRocPH9Yff/yhzZs36/Tp02rdurV1W5I0Y8YMm+3t378/1bneQUFBslgs6b4gc8uWLWrXrp0aNWqk5cuXa/fu3Ro4cKD1uJDuTgM5cOCAGjdurJ9//lklSpTQDz/8IOnBx82jiIuL0/Dhw232e9++fTp69KjNPfdTmn5zLycnp2S3BAUAAFnTUxHE3dzcVKRIERUoUMDmloW7du1SYmKixo8fr8qVKys4OFh///13stf7+/vr3XffVWRkpPr06aMZM2ZY1+XOnVsdO3bUf//7X02aNEn/+c9/Uq2jbdu22r9/v3bt2qVFixZZz/RKUvny5XX27Fk5ODioSJEiNo/UboG4cuVKXbt2Tbt377YJaBEREYqMjNSVK1fk6ekpHx8f7dixw/q6hIQEm/nfhQoVUrZs2WzaxMbG6siRIw8Z2bt1Hzx4MFnNRYoUsd5RxcHBQaGhoRo7dqz27t2rkydPWs/Y3y9//vyqWbOm5s2bp3nz5qlevXrWqRs+Pj7Kmzevjh8/nmxbBQsWTLE/b29vNWjQQF9//bXNBaRJUvuFYPPmzQoICNDAgQNVoUIFBQUFWS+qvVdwcLB69eqltWvX6tVXX7VerCs9+LhJr/Lly+vw4cMpjrOd3VPxzwwAADxlnqo54vcrUqSIbt++rSlTpqhp06batGmTvvnmG5s2H374oRo2bKjg4GBdvnxZv/zyi4oXLy5JGjJkiF544QWVLFlS8fHxWr58uXVdSgIDA1W1alV17txZCQkJevnll63rQkNDVaVKFb3yyisaO3as9ZeCFStWqHnz5ilOOZg1a5YaN26sMmXK2CwvUaKEevXqpXnz5ql79+56//33NXr0aBUpUkTFihXTlClTdPnyZesccg8PD3Xs2FEfffSRvL29lSdPHg0dOlR2dnbWNqnp37+/KleurB49eqhLly5yc3PTwYMHtW7dOn311Vdavny5jh8/rho1aihHjhxauXKlEhMTU5z2kqRdu3YaOnSobt26ZXNxpHR3rnTPnj3l6empsLAwxcfHa+fOnbp8+bJ69+6dYn9ff/21qlWrpooVK2rEiBEqXbq07ty5o3Xr1mnatGk6dOhQstcEBQUpOjpaCxYs0IsvvqgVK1ZYz3ZL0s2bN/XRRx+pZcuWKliwoM6cOaMdO3aoRYsWkh583DyKIUOGqEmTJipQoIBatmwpOzs77dmzR/v379enn376yP0CAICs66k+VVemTBlNmDBBY8aMUalSpTRv3jyNHj3apk1CQoK6d++u4sWLKywsTMHBwdYLOR0dHTVgwACVLl1aNWrUkL29vRYsWPDAbbZr10579uxR8+bNbe4TbbFYtHLlStWoUUNvvvmmgoOD9frrr+vUqVPy8fFJ1s+5c+e0YsUKa/C7l52dnZo3b65Zs2ZJuhuW27Rpow4dOqhKlSrWedz3TmmYMGGCqlSpoiZNmig0NFTVqlWz3iLwQUqXLq1ff/1VR44cUfXq1VWuXDkNGTLEeqGnl5eXIiMjVadOHRUvXlzffPONIiIiVLJkyVT7bNmypXWe/r23WJTuTvmYOXOm5syZo5CQENWsWVPh4eGpnhGX7p7x/+OPP1S7dm316dNHpUqVUr169bR+/fpkt5RM8vLLL6tXr17q0aOHypYtq82bN1svIJXuzrO/ePGiOnTooODgYL322mtq2LCh9ULIBx03j6JBgwZavny51q5dqxdffFGVK1fWxIkTrXfcAQAAuJ/FeJyr5ZApEhMTVbx4cb322msaOXJkim2uX7+ufPnyafz48ercufMTrhBPytWrV+Xp6Sn/D7+XnZOr2eUAAJBlnPy8cab1nfTzOzY29oHXez3VU1OeF6dOndLatWtVs2ZNxcfH66uvvtKJEyes9+aW7n650P/93/+pYsWKio2NtX6BTrNmzcwqGwAAAI+BIP4UsLOzU3h4uPr27SvDMFSqVCn99NNPyeYsf/HFFzp8+LAcHR31wgsv6Lfffkv1QlEAAAA83QjiTwF/f39t2rTpgW3KlSunXbt2PaGKAAAAkNme6os1AQAAgKyKIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJjAwewCADzc/uENlD17drPLAAAAGYgz4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkczC4AwMOVGrpGdk6uZpcBAECqTn7e2OwSnjmcEQcAAABMQBAHAAAATEAQBwAAAExAEAcAAABMQBAHAAAATEAQBwAAAExAEAcAAABMQBAHAAAATEAQBwAAAExAEAcAAABMQBAHAAAATEAQBwAAAExAEAcAAABMQBAHAAAATEAQBwAAAExAEAcAAABMQBAHAAAATEAQBwAAAExAEEeG69SpkywWiywWixwdHVWkSBGNGDFCd+7cMbu0BwoMDNSkSZPMLgMAADwnHMwuAFlTWFiY5syZo/j4eK1cuVLdu3dXtmzZNGDAgHT1k5CQIIvFIju7p+N3xqetHgAA8OwiTSBTODk5ydfXVwEBAXrvvfcUGhqqZcuWKT4+Xn379lW+fPnk5uamSpUqacOGDdbXhYeHy8vLS8uWLVOJEiXk5OSk6OhoxcfHq3///vL395eTk5OKFCmiWbNmWV+3f/9+NWzYUO7u7vLx8dEbb7yhCxcuWNfXqlVLPXr0UI8ePeTp6alcuXJp8ODBMgzDuv7UqVPq1auX9Wz+g+q5fPmyOnTooBw5csjV1VUNGzbU0aNHk+3HmjVrVLx4cbm7uyssLEwxMTGZPPIAAOBZQRDHE+Hi4qJbt26pR48e2rJlixYsWKC9e/eqVatWCgsLswmxN27c0JgxYzRz5kwdOHBAefLkUYcOHRQREaEvv/xShw4d0vTp0+Xu7i5JunLliurUqaNy5cpp586dWr16tc6dO6fXXnvNpoa5c+fKwcFB27dv1+TJkzVhwgTNnDlTkhQZGan8+fNrxIgRiomJsQnMKdXTqVMn7dy5U8uWLdOWLVtkGIYaNWqk27dv27zuiy++0HfffaeNGzcqOjpaffv2zcxhBgAAzxCmpiBTGYah9evXa82aNWrTpo3mzJmj6Oho5c2bV5LUt29frV69WnPmzNGoUaMkSbdv39bUqVNVpkwZSdKRI0f0/fffa926dQoNDZUkFSpUyLqNr776SuXKlbO+XpJmz54tf39/HTlyRMHBwZIkf39/TZw4URaLRUWLFtW+ffs0ceJEde3aVd7e3rK3t5eHh4d8fX1t9uH+eo4ePaply5Zp06ZNqlq1qiRp3rx58vf315IlS9SqVSvr67755hsVLlxYktSjRw+NGDHigeMVHx+v+Ph46/OrV6+mZ7gBAMAzhDPiyBTLly+Xu7u7nJ2d1bBhQ7Vu3VotW7ZUQkKCgoOD5e7ubn38+uuvOnbsmPW1jo6OKl26tPV5VFSU7O3tVbNmzRS3tWfPHv3yyy82fRYrVkySbPqtXLmydcqJJFWpUkVHjx5VQkLCA/fl/noOHTokBwcHVapUybosZ86cKlq0qA4dOmRd5urqag3hkuTn56fz588/cFujR4+Wp6en9eHv7//A9gAA4NnFGXFkitq1a2vatGlydHRU3rx55eDgoP/973+yt7fXrl27ZG9vb9M+aZqJdHcay72B2cXF5YHbiouLU9OmTTVmzJhk6/z8/B5zT5LXk1bZsmWzeW6xWKxz0lMzYMAA9e7d2/r86tWrhHEAALIogjgyhZubm4oUKWKzrFy5ckpISND58+dVvXr1NPcVEhKixMRE/frrr9apKfcqX768Fi9erMDAQDk4pH5Ib9u2zeb51q1bFRQUZP2lwNHR8aFnxyWpePHiunPnjrZt22admnLx4kUdPnxYJUqUSPN+pcTJyUlOTk6P1QcAAHg2MDUFT0xwcLDatWunDh06KDIyUidOnND27ds1evRorVixItXXBQYGqmPHjnrrrbe0ZMkSnThxQhs2bND3338vSerevbsuXbqkNm3aaMeOHTp27JjWrFmjN9980yZYR0dHq3fv3jp8+LAiIiI0ZcoUffDBBzbb2bhxo/766y+bO67cLygoSM2aNVPXrl31+++/a8+ePWrfvr3y5cunZs2aZcBIAQCA5wFBHE/UnDlz1KFDB/Xp00dFixbVK6+8oh07dqhAgQIPfN20adPUsmVLdevWTcWKFVPXrl11/fp1SVLevHm1adMmJSQkqH79+goJCdGHH34oLy8vm/t9d+jQQTdv3lTFihXVvXt3ffDBB3r77bet60eMGKGTJ0+qcOHCyp0790P344UXXlCTJk1UpUoVGYahlStXJpuOAgAAkBqL8bBJq0AWUKtWLZUtW/aZ++bMq1ev3r1o88PvZefkanY5AACk6uTnjc0u4amR9PM7NjZW2bNnT7UdZ8QBAAAAExDEAQAAABNw1xQ8FzZs2GB2CQAAADY4Iw4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmIAgDgAAAJiAIA4AAACYgCAOAAAAmMDB7AIAPNz+4Q2UPXt2s8sAAAAZiDPiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACQjiAAAAgAkI4gAAAIAJCOIAAACACRzMLgBA6gzDkCRdvXrV5EoAAEBaJf3cTvo5nhqCOPAUu3jxoiTJ39/f5EoAAEB6Xbt2TZ6enqmuJ4gDTzFvb29JUnR09AP/IWdlV69elb+/v06fPq3s2bObXY4pGIO7GAfGQGIMJMYgydM8DoZh6Nq1a8qbN+8D2xHEgaeYnd3dyzg8PT2fug+ZJy179uyMAWMgiXGQGAOJMZAYgyRP6zik5QQaF2sCAAAAJiCIAwAAACYgiANPMScnJw0dOlROTk5ml2IaxoAxSMI4MAYSYyAxBkmywjhYjIfdVwUAAABAhuOMOAAAAGACgjgAAABgAoI4AAAAYAKCOAAAAGACgjjwBH399dcKDAyUs7OzKlWqpO3btz+w/cKFC1WsWDE5OzsrJCREK1eutFlvGIaGDBkiPz8/ubi4KDQ0VEePHs3MXcgQGT0OnTp1ksVisXmEhYVl5i48tvSMwYEDB9SiRQsFBgbKYrFo0qRJj93n0yCjx2DYsGHJjoNixYpl4h48vvSMwYwZM1S9enXlyJFDOXLkUGhoaLL2z8NnQlrGIat/JkRGRqpChQry8vKSm5ubypYtq++++86mzbN4LGT0GDwTx4EB4IlYsGCB4ejoaMyePds4cOCA0bVrV8PLy8s4d+5ciu03bdpk2NvbG2PHjjUOHjxoDBo0yMiWLZuxb98+a5vPP//c8PT0NJYsWWLs2bPHePnll42CBQsaN2/efFK7lW6ZMQ4dO3Y0wsLCjJiYGOvj0qVLT2qX0i29Y7B9+3ajb9++RkREhOHr62tMnDjxsfs0W2aMwdChQ42SJUvaHAf//PNPJu/Jo0vvGLRt29b4+uuvjd27dxuHDh0yOnXqZHh6ehpnzpyxtnkePhPSMg5Z/TPhl19+MSIjI42DBw8af/75pzFp0iTD3t7eWL16tbXNs3YsZMYYPAvHAUEceEIqVqxodO/e3fo8ISHByJs3rzF69OgU27/22mtG48aNbZZVqlTJeOeddwzDMIzExETD19fXGDdunHX9lStXDCcnJyMiIiIT9iBjZPQ4GMbdD9tmzZplSr2ZIb1jcK+AgIAUQ+jj9GmGzBiDoUOHGmXKlMnAKjPX475nd+7cMTw8PIy5c+cahvH8fCbc7/5xMIzn6zMhSbly5YxBgwYZhvFsHgsZPQaG8WwcB0xNAZ6AW7duadeuXQoNDbUus7OzU2hoqLZs2ZLia7Zs2WLTXpIaNGhgbX/ixAmdPXvWpo2np6cqVaqUap9my4xxSLJhwwblyZNHRYsW1XvvvaeLFy9m/A5kgEcZAzP6zEyZWe/Ro0eVN29eFSpUSO3atVN0dPTjlpspMmIMbty4odu3b8vb21vS8/OZcL/7xyHJ8/KZYBiG1q9fr8OHD6tGjRqSnr1jITPGIMnTfhwQxIEn4MKFC0pISJCPj4/Nch8fH509ezbF15w9e/aB7ZP+TE+fZsuMcZCksLAwffvtt1q/fr3GjBmjX3/9VQ0bNlRCQkLG78RjepQxMKPPzJRZ9VaqVEnh4eFavXq1pk2bphMnTqh69eq6du3a45ac4TJiDPr376+8efNaw8vz8plwv/vHQXo+PhNiY2Pl7u4uR0dHNW7cWFOmTFG9evUkPXvHQmaMgfRsHAcOZhcAAI/r9ddft/49JCREpUuXVuHChbVhwwbVrVvXxMrwJDVs2ND699KlS6tSpUoKCAjQ999/r86dO5tYWcb7/PPPtWDBAm3YsEHOzs5ml2Oa1MbhefhM8PDwUFRUlOLi4rR+/Xr17t1bhQoVUq1atcwu7Yl52Bg8C8cBZ8SBJyBXrlyyt7fXuXPnbJafO3dOvr6+Kb7G19f3ge2T/kxPn2bLjHFISaFChZQrVy79+eefj190BnuUMTCjz8z0pOr18vJScHBwljsOvvjiC33++edau3atSpcubV3+vHwmJEltHFKSFT8T7OzsVKRIEZUtW1Z9+vRRy5YtNXr0aEnP3rGQGWOQkqfxOCCIA0+Ao6OjXnjhBa1fv966LDExUevXr1eVKlVSfE2VKlVs2kvSunXrrO0LFiwoX19fmzZXr17Vtm3bUu3TbJkxDik5c+aMLl68KD8/v4wpPAM9yhiY0WdmelL1xsXF6dixY1nqOBg7dqxGjhyp1atXq0KFCjbrnpfPBOnB45CS5+EzITExUfHx8ZKevWMhM8YgJU/lcWD21aLA82LBggWGk5OTER4ebhw8eNB4++23DS8vL+Ps2bOGYRjGG2+8YXz88cfW9ps2bTIcHByML774wjh06JAxdOjQFG9f6OXlZSxdutTYu3ev0axZs6f69lSGkfHjcO3aNaNv377Gli1bjBMnThg//fSTUb58eSMoKMj4999/TdnHh0nvGMTHxxu7d+82du/ebfj5+Rl9+/Y1du/ebRw9ejTNfT5tMmMM+vTpY2zYsME4ceKEsWnTJiM0NNTIlSuXcf78+Se+f2mR3jH4/PPPDUdHR2PRokU2t2O7du2aTZus/pnwsHF4Hj4TRo0aZaxdu9Y4duyYcfDgQeOLL74wHBwcjBkzZljbPGvHQkaPwbNyHBDEgSdoypQpRoECBQxHR0ejYsWKxtatW63ratasaXTs2NGm/ffff28EBwcbjo6ORsmSJY0VK1bYrE9MTDQGDx5s+Pj4GE5OTkbdunWNw4cPP4ldeSwZOQ43btww6tevb+TOndvIli2bERAQYHTt2vWpDaBJ0jMGJ06cMCQle9SsWTPNfT6NMnoMWrdubfj5+RmOjo5Gvnz5jNatWxt//vnnE9yj9EvPGAQEBKQ4BkOHDrW2eR4+Ex42Ds/DZ8LAgQONIkWKGM7OzkaOHDmMKlWqGAsWLLDp71k8FjJyDJ6V48BiGIbxZM/BAwAAAGCOOAAAAGACgjgAAABgAoI4AAAAYAKCOAAAAGACgjgAAABgAoI4AAAAYAKCOAAAAGACgjgA4Lk1bNgwWSyWZI9SpUpZ26xbt05t27ZV4cKFZbFY1KNHDxMrBpCVOJhdAAAAZnJxcdHPP/9ss8zV1dX699WrV2vPnj2qWbOmLl269KTLA5CFEcQBAM81Ozs7Va5cOdX148aN0/jx4yUpWWB/Wt28eVMuLi5mlwHgIZiaAgDAA9jZPdqPyitXrqhr167Kly+fnJ2d5e/vr9dff92mzV9//aUOHTrIx8dHLi4uKlasmCZPnmxdn5iYqE8//VSBgYFycnJSsWLFNH36dJs+hg0bJnd3d23fvl1VqlSRs7Ozvv76a0nSoUOH1KxZM3l6esrNzU2NGzfWsWPHHml/AGQ8zogDAJ57d+7csXlub28vi8XyWH327t1bq1at0ueff67AwEDFxMRo1apV1vUXL15UlSpVJEmfffaZChUqpKNHj9oE5Y8++kiTJ0/WoEGDVLVqVS1fvlzvvvuubt++bTNX/datW2rbtq169eqlUaNGKWfOnDp+/LiqVq2qUqVKKTw8XHZ2dvrss89Ut25dHT58WE5OTo+1fwAeH0EcAPBcu379urJly2az7LvvvlP79u0fq9/t27erbdu26tixo3XZvWfEJ0yYoPPnz+v//u//FBgYKEmqU6eOdf2FCxc0ZcoUffTRRxo2bJgkqX79+rpw4YJGjBih9957T/b29pKk27dv67PPPlPr1q2tr+/YsaO8vb21bt06OTs7S5KqVq2qQoUKadasWerWrdtj7R+Ax0cQBwA811xcXLRx40abZYUKFXrsfsuXL6/w8HD5+fkpLCzM5k4skrR+/XrVqVPHGsLvt23bNt2+fVutWrWyWd66dWtFREToyJEjKl68uHV548aNbdqtXbtWr7/+uhwcHKxn/HPkyKFy5cppx44dj71/AB4fQRwA8Fyzs7NThQoVMrzfKVOmyNvbW+PHj9dHH30kf39/DRgwQO+9956ku1NT7g/n97p8+bIkycfHx2Z50vN77+Di6uoqd3d3m3YXLlzQpEmTNGnSpGR9Ozo6PtI+AchYBHEAADKBp6enNQjv27dPkydPVrdu3VSqVClVr15dOXPm1N9//53q6729vSVJ58+fV758+azLz507Z7NeUorz2b29vdW4ceMUp6B4eHg88n4ByDjcNQUAgEwWEhKiiRMnSrp7JxNJCg0N1c8//6zo6OgUX1OxYkVly5ZNCxcutFn+/fffK0+ePAoODn7gNkNDQ7V//36VK1dOFSpUsHkULVo0A/YKwOPijDgAAA9w6tQp65zqGzdu6NixY1q0aJEkqWXLlqm+rlq1amrevLlKlSole3t7ffvtt3J0dFT16tUlSb169dK3336rGjVqaPDgwSpUqJCOHz+uI0eOaMyYMcqVK5fef/99jRs3Ts7OzqpcubJWrlyp+fPna8qUKdYLNVMzfPhwvfjii2rQoIHefvtt+fj46OzZs/r1119VvXp1tWnTJoNGCMCjIogDAPAAv/zyi958803r89WrV2v16tWSJMMwUn1dtWrV9O233+rEiROys7NTSEiIfvzxR+sFljlz5tSmTZs0YMAA9evXTzdu3FBgYKDNVJJx48bJy8tLM2fOtN5P/JtvvtE777zz0LqLFCmi7du3a9CgQerWrZvi4uLk5+enGjVqqHTp0o86HAAykMV40KcIAAAAgEzBHHEAAADABARxAAAAwAQEcQAAAMAEBHEAAADABARxAAAAwAQEcQAAAMAEBHEAAADABARxAAAAwAQEcQAAAMAEBHEAAADABARxAAAAwAQEcQAAAMAE/x8nzQ/H65AbEAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Based on F1-score metrics, the best model is Passive Aggresive Classifier with macro average F1-score of 0.36\n",
        "\n",
        "print(classification_report(y_pred=pa.predict(X_test), y_true=y_test, labels=new_classes))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wfI0SKFhln9K",
        "outputId": "5f5d8fd9-93e1-45f1-b05f-0a325d2d5422"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "      B-DATE       0.26      0.12      0.17        49\n",
            "  B-LOCATION       0.55      0.21      0.30        78\n",
            "       B-ORG       0.52      0.50      0.51       132\n",
            "    B-PERSON       0.78      0.68      0.72       210\n",
            "      I-DATE       0.29      0.13      0.18        60\n",
            "  I-LOCATION       0.43      0.07      0.12        42\n",
            "       I-ORG       0.47      0.47      0.47       110\n",
            "    I-PERSON       0.63      0.31      0.42        87\n",
            "\n",
            "   micro avg       0.58      0.42      0.49       768\n",
            "   macro avg       0.49      0.31      0.36       768\n",
            "weighted avg       0.56      0.42      0.46       768\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "irxJS9fWn18x"
      },
      "source": [
        "**BERT**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "FUb2WvRLngOQ"
      },
      "outputs": [],
      "source": [
        "from flair.data import Corpus\n",
        "from flair.datasets import ColumnCorpus"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load train, validation, test word label datasets\n",
        "\n",
        "train = pd.read_csv('data/ner/train.csv')\n",
        "train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "c2k-I-SqqNAz",
        "outputId": "295a5608-0b07-4b51-bd22-5aae28c98e1d"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        message_id  \\\n",
              "0    <11792830.1075859955495.JavaMail.evans@thyme>   \n",
              "1    <16461482.1075846947944.JavaMail.evans@thyme>   \n",
              "2    <27347038.1075842439828.JavaMail.evans@thyme>   \n",
              "3    <22015863.1075844739388.JavaMail.evans@thyme>   \n",
              "4    <27358009.1075862249772.JavaMail.evans@thyme>   \n",
              "..                                             ...   \n",
              "763  <23887193.1075840960293.JavaMail.evans@thyme>   \n",
              "764  <12852426.1075844351646.JavaMail.evans@thyme>   \n",
              "765    <685271.1075852754266.JavaMail.evans@thyme>   \n",
              "766   <3384182.1075849303276.JavaMail.evans@thyme>   \n",
              "767   <2826937.1075862373832.JavaMail.evans@thyme>   \n",
              "\n",
              "                                                  text  \\\n",
              "0    Good catch.  NY law should apply throughout th...   \n",
              "1    Tana,\\nWow, you are really on the ball.  Looks...   \n",
              "2    I am very pleased to announce that L.D. Stephe...   \n",
              "3    Chris:\\n\\nWe should chat.  Per the ISDA, ENA i...   \n",
              "4    I have made additional changes on pages 11-12 ...   \n",
              "..                                                 ...   \n",
              "763  We plan to have a conference call at 2pm Sunda...   \n",
              "764  Attached in WordPerfect are blacklines of the ...   \n",
              "765  We intend to start officializing a gas book fo...   \n",
              "766  We are very close to sending out a redline.? A...   \n",
              "767  The attached press release is the latest draft...   \n",
              "\n",
              "                                              entities  \\\n",
              "0    [{'word': 'Leonardo Pacheco', 'tag': 'PERSON'}...   \n",
              "1    [{'word': 'Tana', 'tag': 'PERSON'}, {'word': '...   \n",
              "2    [{'word': 'L.D. Stephens', 'tag': 'PERSON'}, {...   \n",
              "3    [{'word': 'Chris H Foster', 'tag': 'PERSON'}, ...   \n",
              "4    [{'word': 'Mark Mayer', 'tag': 'PERSON'}, {'wo...   \n",
              "..                                                 ...   \n",
              "763  [{'word': 'David', 'tag': 'PERSON'}, {'word': ...   \n",
              "764  [{'word': 'Deborah Lowe', 'tag': 'PERSON'}, {'...   \n",
              "765  [{'word': 'Kathy', 'tag': 'PERSON'}, {'word': ...   \n",
              "766  [{'word': 'Linda Y. Sherif', 'tag': 'PERSON'},...   \n",
              "767  [{'word': 'Enron', 'tag': 'PERSON'}, {'word': ...   \n",
              "\n",
              "                                            annotation  \n",
              "0    [('Leonardo Pacheco', 'PERSON'), ('NY', 'LOCAT...  \n",
              "1    [('Tana', 'PERSON'), ('Matthias', 'PERSON'), (...  \n",
              "2    [('L.D. Stephens', 'PERSON'), ('Enron', 'PERSO...  \n",
              "3    [('Chris H Foster', 'PERSON'), ('Sara Shacklet...  \n",
              "4    [('Mark Mayer', 'PERSON'), ('BPA', 'PERSON'), ...  \n",
              "..                                                 ...  \n",
              "763  [('David', 'PERSON'), ('Tammie', 'PERSON'), ('...  \n",
              "764  [('Deborah Lowe', 'PERSON'), ('1717 Main Stree...  \n",
              "765  [('Kathy', 'PERSON'), ('McKay', 'PERSON'), ('J...  \n",
              "766  [('Linda Y. Sherif', 'PERSON'), ('120 Montgome...  \n",
              "767  [('Enron', 'PERSON'), ('Citigroup', 'PERSON'),...  \n",
              "\n",
              "[768 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f2101ec1-fa74-4837-b396-b993ce55b7c7\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message_id</th>\n",
              "      <th>text</th>\n",
              "      <th>entities</th>\n",
              "      <th>annotation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>&lt;11792830.1075859955495.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Good catch.  NY law should apply throughout th...</td>\n",
              "      <td>[{'word': 'Leonardo Pacheco', 'tag': 'PERSON'}...</td>\n",
              "      <td>[('Leonardo Pacheco', 'PERSON'), ('NY', 'LOCAT...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;16461482.1075846947944.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Tana,\\nWow, you are really on the ball.  Looks...</td>\n",
              "      <td>[{'word': 'Tana', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Tana', 'PERSON'), ('Matthias', 'PERSON'), (...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>&lt;27347038.1075842439828.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I am very pleased to announce that L.D. Stephe...</td>\n",
              "      <td>[{'word': 'L.D. Stephens', 'tag': 'PERSON'}, {...</td>\n",
              "      <td>[('L.D. Stephens', 'PERSON'), ('Enron', 'PERSO...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>&lt;22015863.1075844739388.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Chris:\\n\\nWe should chat.  Per the ISDA, ENA i...</td>\n",
              "      <td>[{'word': 'Chris H Foster', 'tag': 'PERSON'}, ...</td>\n",
              "      <td>[('Chris H Foster', 'PERSON'), ('Sara Shacklet...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>&lt;27358009.1075862249772.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I have made additional changes on pages 11-12 ...</td>\n",
              "      <td>[{'word': 'Mark Mayer', 'tag': 'PERSON'}, {'wo...</td>\n",
              "      <td>[('Mark Mayer', 'PERSON'), ('BPA', 'PERSON'), ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>763</th>\n",
              "      <td>&lt;23887193.1075840960293.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>We plan to have a conference call at 2pm Sunda...</td>\n",
              "      <td>[{'word': 'David', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('David', 'PERSON'), ('Tammie', 'PERSON'), ('...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>764</th>\n",
              "      <td>&lt;12852426.1075844351646.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Attached in WordPerfect are blacklines of the ...</td>\n",
              "      <td>[{'word': 'Deborah Lowe', 'tag': 'PERSON'}, {'...</td>\n",
              "      <td>[('Deborah Lowe', 'PERSON'), ('1717 Main Stree...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>765</th>\n",
              "      <td>&lt;685271.1075852754266.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>We intend to start officializing a gas book fo...</td>\n",
              "      <td>[{'word': 'Kathy', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Kathy', 'PERSON'), ('McKay', 'PERSON'), ('J...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>766</th>\n",
              "      <td>&lt;3384182.1075849303276.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>We are very close to sending out a redline.? A...</td>\n",
              "      <td>[{'word': 'Linda Y. Sherif', 'tag': 'PERSON'},...</td>\n",
              "      <td>[('Linda Y. Sherif', 'PERSON'), ('120 Montgome...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>767</th>\n",
              "      <td>&lt;2826937.1075862373832.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>The attached press release is the latest draft...</td>\n",
              "      <td>[{'word': 'Enron', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Enron', 'PERSON'), ('Citigroup', 'PERSON'),...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>768 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f2101ec1-fa74-4837-b396-b993ce55b7c7')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f2101ec1-fa74-4837-b396-b993ce55b7c7 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f2101ec1-fa74-4837-b396-b993ce55b7c7');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3318d4c4-4282-4640-b331-a1381fb36022\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3318d4c4-4282-4640-b331-a1381fb36022')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3318d4c4-4282-4640-b331-a1381fb36022 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val = pd.read_csv('data/ner/val.csv')\n",
        "val"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        },
        "id": "1qTxf4CSqM-s",
        "outputId": "ed5820a7-6425-44b9-8c09-9abed2e064e0"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                       message_id  \\\n",
              "0   <18050053.1075859962876.JavaMail.evans@thyme>   \n",
              "1   <31623156.1075845089208.JavaMail.evans@thyme>   \n",
              "2   <24135778.1075846138474.JavaMail.evans@thyme>   \n",
              "3    <9457688.1075861461211.JavaMail.evans@thyme>   \n",
              "4   <26484226.1075855280232.JavaMail.evans@thyme>   \n",
              "..                                            ...   \n",
              "91  <10689485.1075851883019.JavaMail.evans@thyme>   \n",
              "92  <12019737.1075861047362.JavaMail.evans@thyme>   \n",
              "93  <31192798.1075842099616.JavaMail.evans@thyme>   \n",
              "94  <28609896.1075861988637.JavaMail.evans@thyme>   \n",
              "95  <12936604.1075841300752.JavaMail.evans@thyme>   \n",
              "\n",
              "                                                 text  \\\n",
              "0   Al:\\n\\nPer my voice mail, I have left a messag...   \n",
              "1   To all,\\n\\nI visited with Kam earlier today to...   \n",
              "2   Beth, Mark - \\n\\nI think we are ready to move ...   \n",
              "3   Dear Jason Wolfe,\\n\\nWe are happy to inform yo...   \n",
              "4   Scheduling Issue:\\t\\n\\n\\t1024\\tMIRANT(Rala) to...   \n",
              "..                                                ...   \n",
              "91  ?\\n?\\n?\\n?\\n?\\n?\\n?\\n?\\nHi all, Merry Christma...   \n",
              "92  March 13, 2002\\nNotice No. NMS-22\\n\\n\\nNOTICE ...   \n",
              "93  Steve:\\nI am faxing you our remaining comments...   \n",
              "94  Mark,\\n\\nI spoke to the postmaster and she sta...   \n",
              "95  Start Date: 1/20/02; HourAhead hour: 3;  HourA...   \n",
              "\n",
              "                                             entities  \\\n",
              "0   [{'word': 'Al', 'tag': 'PERSON'}, {'word': 'Ma...   \n",
              "1   [{'word': 'Kam', 'tag': 'PERSON'}, {'word': 'P...   \n",
              "2   [{'word': 'Beth', 'tag': 'PERSON'}, {'word': '...   \n",
              "3   [{'word': 'Jason Wolfe', 'tag': 'PERSON'}, {'w...   \n",
              "4   [{'word': 'Rala', 'tag': 'PERSON'}, {'word': '...   \n",
              "..                                                ...   \n",
              "91  [{'word': 'Diane', 'tag': 'PERSON'}, {'word': ...   \n",
              "92  [{'word': 'Michael Cokeley', 'tag': 'PERSON'},...   \n",
              "93  [{'word': 'Steve', 'tag': 'PERSON'}, {'word': ...   \n",
              "94  [{'word': 'Mark', 'tag': 'PERSON'}, {'word': '...   \n",
              "95               [{'word': '1/20/02', 'tag': 'DATE'}]   \n",
              "\n",
              "                                           annotation  \n",
              "0   [('Al', 'PERSON'), ('Mark', 'PERSON'), ('Alfre...  \n",
              "1   [('Kam', 'PERSON'), ('Pete', 'PERSON'), ('Scot...  \n",
              "2   [('Beth', 'PERSON'), ('Mark', 'PERSON'), ('Ste...  \n",
              "3   [('Jason Wolfe', 'PERSON'), ('Sergei', 'PERSON...  \n",
              "4   [('Rala', 'PERSON'), ('Katherine', 'PERSON'), ...  \n",
              "..                                                ...  \n",
              "91  [('Diane', 'PERSON'), ('Christmas', 'DATE'), (...  \n",
              "92  [('Michael Cokeley', 'PERSON'), ('David Kimmel...  \n",
              "93  [('Steve', 'PERSON'), ('DB', 'PERSON'), ('Caro...  \n",
              "94       [('Mark', 'PERSON'), ('Veronica', 'PERSON')]  \n",
              "95                              [('1/20/02', 'DATE')]  \n",
              "\n",
              "[96 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f24572cd-7394-4e11-88d9-b3261adc2e09\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message_id</th>\n",
              "      <th>text</th>\n",
              "      <th>entities</th>\n",
              "      <th>annotation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>&lt;18050053.1075859962876.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Al:\\n\\nPer my voice mail, I have left a messag...</td>\n",
              "      <td>[{'word': 'Al', 'tag': 'PERSON'}, {'word': 'Ma...</td>\n",
              "      <td>[('Al', 'PERSON'), ('Mark', 'PERSON'), ('Alfre...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;31623156.1075845089208.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>To all,\\n\\nI visited with Kam earlier today to...</td>\n",
              "      <td>[{'word': 'Kam', 'tag': 'PERSON'}, {'word': 'P...</td>\n",
              "      <td>[('Kam', 'PERSON'), ('Pete', 'PERSON'), ('Scot...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>&lt;24135778.1075846138474.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Beth, Mark - \\n\\nI think we are ready to move ...</td>\n",
              "      <td>[{'word': 'Beth', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Beth', 'PERSON'), ('Mark', 'PERSON'), ('Ste...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>&lt;9457688.1075861461211.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Dear Jason Wolfe,\\n\\nWe are happy to inform yo...</td>\n",
              "      <td>[{'word': 'Jason Wolfe', 'tag': 'PERSON'}, {'w...</td>\n",
              "      <td>[('Jason Wolfe', 'PERSON'), ('Sergei', 'PERSON...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>&lt;26484226.1075855280232.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Scheduling Issue:\\t\\n\\n\\t1024\\tMIRANT(Rala) to...</td>\n",
              "      <td>[{'word': 'Rala', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Rala', 'PERSON'), ('Katherine', 'PERSON'), ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>&lt;10689485.1075851883019.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>?\\n?\\n?\\n?\\n?\\n?\\n?\\n?\\nHi all, Merry Christma...</td>\n",
              "      <td>[{'word': 'Diane', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Diane', 'PERSON'), ('Christmas', 'DATE'), (...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>&lt;12019737.1075861047362.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>March 13, 2002\\nNotice No. NMS-22\\n\\n\\nNOTICE ...</td>\n",
              "      <td>[{'word': 'Michael Cokeley', 'tag': 'PERSON'},...</td>\n",
              "      <td>[('Michael Cokeley', 'PERSON'), ('David Kimmel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>&lt;31192798.1075842099616.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Steve:\\nI am faxing you our remaining comments...</td>\n",
              "      <td>[{'word': 'Steve', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Steve', 'PERSON'), ('DB', 'PERSON'), ('Caro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>&lt;28609896.1075861988637.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Mark,\\n\\nI spoke to the postmaster and she sta...</td>\n",
              "      <td>[{'word': 'Mark', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Mark', 'PERSON'), ('Veronica', 'PERSON')]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>&lt;12936604.1075841300752.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Start Date: 1/20/02; HourAhead hour: 3;  HourA...</td>\n",
              "      <td>[{'word': '1/20/02', 'tag': 'DATE'}]</td>\n",
              "      <td>[('1/20/02', 'DATE')]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>96 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f24572cd-7394-4e11-88d9-b3261adc2e09')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f24572cd-7394-4e11-88d9-b3261adc2e09 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f24572cd-7394-4e11-88d9-b3261adc2e09');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2fa2a836-2ffa-4fcb-82cc-a2c4ca0483a1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2fa2a836-2ffa-4fcb-82cc-a2c4ca0483a1')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2fa2a836-2ffa-4fcb-82cc-a2c4ca0483a1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test = pd.read_csv('data/ner/test.csv')\n",
        "test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "oQL80BvltJT5",
        "outputId": "974169b2-07ca-4da3-ced2-d86be5091a64"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                       message_id  \\\n",
              "0   <15886210.1075854965508.JavaMail.evans@thyme>   \n",
              "1   <17217905.1075853978006.JavaMail.evans@thyme>   \n",
              "2   <28372483.1075843343800.JavaMail.evans@thyme>   \n",
              "3    <7553175.1075863444700.JavaMail.evans@thyme>   \n",
              "4   <16687289.1075862330552.JavaMail.evans@thyme>   \n",
              "..                                            ...   \n",
              "91  <17780105.1075840540168.JavaMail.evans@thyme>   \n",
              "92  <16492427.1075847631885.JavaMail.evans@thyme>   \n",
              "93  <11901989.1075844026014.JavaMail.evans@thyme>   \n",
              "94  <17922592.1075839996237.JavaMail.evans@thyme>   \n",
              "95  <26340375.1075840808386.JavaMail.evans@thyme>   \n",
              "\n",
              "                                                 text  \\\n",
              "0   Allright fellas here's the gameplan . . . I'll...   \n",
              "1   Tina,\\nKoch never returned my calls. Based on ...   \n",
              "2   I wanted to follow-up with everyone following ...   \n",
              "3   Vince,\\n\\n  I agree with you that it's a lesso...   \n",
              "4   I trust everything is going well.  Please let ...   \n",
              "..                                                ...   \n",
              "91  Heidi,\\n\\nI have no way of being able to discu...   \n",
              "92  Teh rest of the parts for Phil's PC arrived to...   \n",
              "93  Hey guys thanks a million for supporting the g...   \n",
              "94  Keoni,\\nI am not concerned with the improperly...   \n",
              "95  I will be circulating in the next few minutes ...   \n",
              "\n",
              "                                             entities  \\\n",
              "0   [{'word': 'Kevin A. Boone', 'tag': 'PERSON'}, ...   \n",
              "1   [{'word': 'Tina', 'tag': 'PERSON'}, {'word': '...   \n",
              "2   [{'word': 'Steve M', 'tag': 'PERSON'}, {'word'...   \n",
              "3   [{'word': 'Vince', 'tag': 'PERSON'}, {'word': ...   \n",
              "4   [{'word': 'next Board meeting', 'tag': 'DATE'}...   \n",
              "..                                                ...   \n",
              "91  [{'word': 'Heidi', 'tag': 'PERSON'}, {'word': ...   \n",
              "92  [{'word': 'Phil', 'tag': 'PERSON'}, {'word': '...   \n",
              "93  [{'word': 'Next Wednesday August 16th', 'tag':...   \n",
              "94  [{'word': 'Keoni', 'tag': 'PERSON'}, {'word': ...   \n",
              "95  [{'word': 'Cris Sherman', 'tag': 'PERSON'}, {'...   \n",
              "\n",
              "                                           annotation  \n",
              "0   [('Kevin A. Boone', 'PERSON'), ('Matt', 'PERSO...  \n",
              "1   [('Tina', 'PERSON'), ('Koch', 'PERSON'), ('Cyn...  \n",
              "2   [('Steve M', 'PERSON'), ('Janine', 'PERSON'), ...  \n",
              "3   [('Vince', 'PERSON'), ('Frank', 'PERSON'), ('S...  \n",
              "4   [('next Board meeting', 'DATE'), ('2001', 'DAT...  \n",
              "..                                                ...  \n",
              "91  [('Heidi', 'PERSON'), ('Randall Chapman', 'PER...  \n",
              "92  [('Phil', 'PERSON'), ('Kathy', 'PERSON'), ('Al...  \n",
              "93  [('Next Wednesday August 16th', 'DATE'), ('41s...  \n",
              "94  [('Keoni', 'PERSON'), ('Bill', 'PERSON'), ('CA...  \n",
              "95  [('Cris Sherman', 'PERSON'), ('Theresa Vos', '...  \n",
              "\n",
              "[96 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-67ac907f-79a0-4965-9e43-2fef63534afe\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message_id</th>\n",
              "      <th>text</th>\n",
              "      <th>entities</th>\n",
              "      <th>annotation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>&lt;15886210.1075854965508.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Allright fellas here's the gameplan . . . I'll...</td>\n",
              "      <td>[{'word': 'Kevin A. Boone', 'tag': 'PERSON'}, ...</td>\n",
              "      <td>[('Kevin A. Boone', 'PERSON'), ('Matt', 'PERSO...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;17217905.1075853978006.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Tina,\\nKoch never returned my calls. Based on ...</td>\n",
              "      <td>[{'word': 'Tina', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Tina', 'PERSON'), ('Koch', 'PERSON'), ('Cyn...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>&lt;28372483.1075843343800.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I wanted to follow-up with everyone following ...</td>\n",
              "      <td>[{'word': 'Steve M', 'tag': 'PERSON'}, {'word'...</td>\n",
              "      <td>[('Steve M', 'PERSON'), ('Janine', 'PERSON'), ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>&lt;7553175.1075863444700.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Vince,\\n\\n  I agree with you that it's a lesso...</td>\n",
              "      <td>[{'word': 'Vince', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Vince', 'PERSON'), ('Frank', 'PERSON'), ('S...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>&lt;16687289.1075862330552.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I trust everything is going well.  Please let ...</td>\n",
              "      <td>[{'word': 'next Board meeting', 'tag': 'DATE'}...</td>\n",
              "      <td>[('next Board meeting', 'DATE'), ('2001', 'DAT...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>&lt;17780105.1075840540168.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Heidi,\\n\\nI have no way of being able to discu...</td>\n",
              "      <td>[{'word': 'Heidi', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Heidi', 'PERSON'), ('Randall Chapman', 'PER...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>&lt;16492427.1075847631885.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Teh rest of the parts for Phil's PC arrived to...</td>\n",
              "      <td>[{'word': 'Phil', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Phil', 'PERSON'), ('Kathy', 'PERSON'), ('Al...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>&lt;11901989.1075844026014.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Hey guys thanks a million for supporting the g...</td>\n",
              "      <td>[{'word': 'Next Wednesday August 16th', 'tag':...</td>\n",
              "      <td>[('Next Wednesday August 16th', 'DATE'), ('41s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>&lt;17922592.1075839996237.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Keoni,\\nI am not concerned with the improperly...</td>\n",
              "      <td>[{'word': 'Keoni', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Keoni', 'PERSON'), ('Bill', 'PERSON'), ('CA...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>&lt;26340375.1075840808386.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I will be circulating in the next few minutes ...</td>\n",
              "      <td>[{'word': 'Cris Sherman', 'tag': 'PERSON'}, {'...</td>\n",
              "      <td>[('Cris Sherman', 'PERSON'), ('Theresa Vos', '...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>96 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-67ac907f-79a0-4965-9e43-2fef63534afe')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-67ac907f-79a0-4965-9e43-2fef63534afe button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-67ac907f-79a0-4965-9e43-2fef63534afe');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0172b59b-c41f-4701-865a-8a432b0ba3bc\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0172b59b-c41f-4701-865a-8a432b0ba3bc')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0172b59b-c41f-4701-865a-8a432b0ba3bc button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "import ast\n",
        "\n",
        "# Convert list as string to list\n",
        "train['annotation'] = train['annotation'].progress_apply(lambda x: ast.literal_eval(x))\n",
        "val['annotation'] = val['annotation'].progress_apply(lambda x: ast.literal_eval(x))\n",
        "test['annotation'] = test['annotation'].progress_apply(lambda x: ast.literal_eval(x))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aY8NuAAnvC_J",
        "outputId": "bc372a0c-077f-4eaf-8721-5b31aa1ac46b"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 768/768 [00:00<00:00, 14793.84it/s]\n",
            "100%|██████████| 96/96 [00:00<00:00, 9388.92it/s]\n",
            "100%|██████████| 96/96 [00:00<00:00, 3612.11it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from difflib import SequenceMatcher\n",
        "import re\n",
        "import pickle\n",
        "\n",
        "def matcher(string, pattern):\n",
        "    '''\n",
        "    Return the start and end index of any pattern present in the text.\n",
        "    '''\n",
        "    match_list = []\n",
        "    pattern = pattern.strip()\n",
        "    seqMatch = SequenceMatcher(None, string, pattern, autojunk=False)\n",
        "    match = seqMatch.find_longest_match(0, len(string), 0, len(pattern))\n",
        "    if (match.size == len(pattern)):\n",
        "        start = match.a\n",
        "        end = match.a + match.size\n",
        "        match_tup = (start, end)\n",
        "        string = string.replace(pattern, \"X\" * len(pattern), 1)\n",
        "        match_list.append(match_tup)\n",
        "\n",
        "    return match_list, string\n",
        "\n",
        "def mark_sentence(s, match_list):\n",
        "    '''\n",
        "    Marks all the entities in the sentence as per the BIO scheme.\n",
        "    '''\n",
        "    word_dict = {}\n",
        "    for word in s.split():\n",
        "        word_dict[word] = 'O'\n",
        "\n",
        "    for start, end, e_type in match_list:\n",
        "        temp_str = s[start:end]\n",
        "        tmp_list = temp_str.split()\n",
        "        if len(tmp_list) > 1:\n",
        "            word_dict[tmp_list[0]] = 'B-' + e_type\n",
        "            for w in tmp_list[1:]:\n",
        "                word_dict[w] = 'I-' + e_type\n",
        "        else:\n",
        "            word_dict[temp_str] = 'B-' + e_type\n",
        "    return word_dict\n",
        "\n",
        "def isFound(substr, string):\n",
        "    if (string.find(substr) == -1):\n",
        "        return False\n",
        "    return True\n",
        "\n",
        "def clean(text):\n",
        "    \"\"\"\n",
        "    Helper fuction to add a space before the punctuations.\n",
        "    This is for better tokenization.\n",
        "    \"\"\"\n",
        "    filters = [\"!\", \"#\", \"$\", \"%\", \"&\", \"(\", \")\", \"/\", \"*\", \".\", \":\", \";\", \"<\", \"=\", \">\", \"?\", \"@\", \"[\",\n",
        "               \"\\\\\", \"]\", \"_\", \"`\", \"{\", \"}\", \"~\", \"'\", \",\"]\n",
        "    for i in text:\n",
        "        if i in filters:\n",
        "            text = text.replace(i, \" \" + i)\n",
        "    return text\n",
        "\n",
        "def create_data(df):\n",
        "    '''\n",
        "    Function responsible for the creation of data\n",
        "    '''\n",
        "    data = []\n",
        "    for text, annotation in zip(df.text, df.annotation):\n",
        "\n",
        "        # Keep only entities that are found in text\n",
        "        annotation = [\n",
        "            entity for entity in annotation\n",
        "                if isFound(entity[0], text)\n",
        "        ]\n",
        "\n",
        "        text = clean(text)\n",
        "        text_ = text\n",
        "\n",
        "        match_list = []\n",
        "        for i in annotation:\n",
        "            a, text_ = matcher(text, i[0])\n",
        "            if a:\n",
        "                match_list.append((a[0][0], a[0][1], i[1]))\n",
        "\n",
        "        d = mark_sentence(text, match_list)\n",
        "        data.append(d)\n",
        "\n",
        "    return data"
      ],
      "metadata": {
        "id": "WJ71xdZ1vC9G"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert data to IOB format\n",
        "\n",
        "train_ner_data = create_data(train)\n",
        "val_ner_data = create_data(val)\n",
        "test_ner_data = create_data(test)"
      ],
      "metadata": {
        "id": "7J05XwNguYUX"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%pprint\n",
        "train_ner_data[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bq3kNPtYuYSQ",
        "outputId": "2ac0854a-9a60-4ef5-902d-855a98251314"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pretty printing has been turned OFF\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Good': 'O', 'catch': 'O', '.': 'O', 'NY': 'B-LOCATION', 'law': 'O', 'should': 'O', 'apply': 'O', 'throughout': 'O', 'the': 'O', 'Americas': 'B-LOCATION', 'and': 'O', 'in': 'O', 'Japan': 'B-LOCATION', 'We': 'O', 'used': 'O', 'UK': 'O', 'Europe': 'B-LOCATION', 'Singapore': 'B-LOCATION', 'on': 'O', 'theory': 'O', 'counterparties': 'O', 'there': 'O', 'would': 'O', 'be': 'O', 'more': 'O', 'comfortable': 'O', 'with': 'O', 'Uk': 'O', 'I': 'O', \"'m\": 'O', 'not': 'O', 'sure': 'O', 'which': 'O', 'Australians': 'O', 'prefer': 'O', 'What': 'O', 'do': 'O', 'you': 'O', 'think': 'O', 'of': 'O', 'following': 'O', ':': 'O', '16': 'O', 'If': 'O', 'are': 'O', 'accessing': 'O', 'this': 'O', 'website': 'O', 'from': 'O', 'North': 'B-LOCATION', 'America': 'I-LOCATION', ',': 'O', 'South': 'B-LOCATION', 'or': 'O', 'terms': 'O', 'legal': 'O', 'Privacy': 'O', 'statement': 'O', 'shall': 'O', 'governed': 'O', 'by': 'O', 'laws': 'O', 'State': 'O', 'New': 'O', 'York': 'O', ';': 'O', 'if': 'O', 'any': 'O', 'other': 'O', 'location': 'O', 'its': 'O', 'English': 'O', 'Leonardo': 'B-PERSON', 'Pacheco': 'I-PERSON', '@ENRON': 'O', 'Australia': 'B-LOCATION'}"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_ner_data[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-uoEBYcrv5GB",
        "outputId": "ca327ba5-30b7-4f33-bfaf-0e35934c2448"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Al': 'B-PERSON', ':': 'O', 'Per': 'O', 'my': 'O', 'voice': 'O', 'mail': 'O', ',': 'O', 'I': 'O', 'have': 'O', 'left': 'O', 'a': 'O', 'message': 'O', 'for': 'O', 'Mark': 'B-PERSON', 'to': 'O', 'give': 'O', 'his': 'O', 'input': 'O', '.': 'O', 'think': 'O', 'you': 'O', 'should': 'O', 'revise': 'O', 'the': 'O', 'letter': 'O', 'as': 'O', 'follows': 'O', 'subject': 'O', \"'s\": 'O', 'edits': 'O', '\"Please': 'O', 'be': 'O', 'advised': 'O', 'that': 'O', 'effective': 'O', 'July': 'O', '17': 'O', '2000': 'O', 'MGplc': 'B-ORG', 'was': 'O', 'aquired': 'O', 'by': 'O', 'Enron': 'B-ORG', 'group': 'O', 'MG': 'B-ORG', 'London': 'I-ORG', 'Inc': 'I-ORG', 'part': 'O', 'of': 'O', 'Effective': 'O', 'October': 'O', '9': 'O', 'changed': 'O', 'its': 'O', 'name': 'O', '\"Enron': 'O', 'Trading': 'O', 'Services': 'O', 'Inc\"': 'O', 'Also': 'O', 'please': 'O', 'Financial': 'O', 'Status': 'O', '(now': 'O', ')': 'O', 'has': 'O', 'not': 'O', 'If': 'O', 'there': 'O', 'are': 'O', 'any': 'O', 'questions': 'O', 'call': 'O', 'me': 'O', 'at': 'O', '212': 'O', '715': 'O', '5845': 'O', '.\"': 'O', 'Alfred': 'B-PERSON', 'Pennisi': 'I-PERSON', '@MGUSA': 'O'}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_ner_data[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WPv6YvvRv5D7",
        "outputId": "7148263d-de2a-45c1-b17a-04142f2f6752"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Allright': 'O', 'fellas': 'O', 'here': 'O', \"'s\": 'O', 'the': 'O', 'gameplan': 'O', '.': 'O', 'I': 'O', \"'ll\": 'O', 'drive': 'O', 'leave': 'O', 'at': 'O', '8am': 'O', 'tomorrow': 'O', 'morning': 'O', 'we': 'O', 'can': 'O', 'meet': 'O', 'my': 'O', 'house': 'O', 'or': 'O', 'Matt': 'B-PERSON', 'it': 'O', 'doesn': 'O', \"'t\": 'O', 'really': 'O', 'matter': 'O', 'to': 'O', 'me': 'O', 'plan': 'O', 'on': 'O', 'leaving': 'O', 'BR': 'O', 'no': 'O', 'later': 'O', 'than': 'O', '9am': 'O', 'sunday': 'O', 'morn': 'O', 'Geaux': 'O', 'Tigers': 'O', 'Regards': 'O', ',': 'O', 'Kevin': 'O', 'A': 'O', 'Boone': 'O', 'Accenture': 'B-ORG', 'Houston': 'O', 'Consultant': 'O', 'Energy': 'O', '(Octel': 'O', ')': 'O', '713': 'O', '.837': 'O', '.1638': 'O', '(Client': 'O', '281': 'O', '.848': 'O', '.1619': 'O', '(C': 'O', '.306': 'O', '.7940': 'O', '(H': 'O', '.864': 'O', '.4149': 'O', '.A': 'O', '.Boone': 'O', '@accenture': 'O', '.com': 'O', 'Our': 'O', 'web': 'O', 'address': 'O', 'is': 'O', 'http': 'O', ':': 'O', '/': 'O', '/www': 'O', '.accenture': 'O', 'This': 'O', 'message': 'O', 'for': 'O', 'designated': 'O', 'recipient': 'O', 'only': 'O', 'and': 'O', 'may': 'O', 'contain': 'O', 'privileged': 'O', 'confidential': 'O', 'information': 'O', 'If': 'O', 'you': 'O', 'have': 'O', 'received': 'O', 'in': 'O', 'error': 'O', 'please': 'O', 'notify': 'O', 'sender': 'O', 'immediately': 'O', 'delete': 'O', 'original': 'O', 'Any': 'O', 'other': 'O', 'use': 'O', 'of': 'O', 'email': 'O', 'by': 'O', 'prohibited': 'O'}"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create utility function to write annotated text to .txt file format\n",
        "\n",
        "def dump_to_txt_file(data, filepath):\n",
        "    \"\"\"\n",
        "    Dump data to txt file\n",
        "\n",
        "    :param data     : data to save\n",
        "    :param filepath : output filepath name\n",
        "    \"\"\"\n",
        "    with open(filepath, 'w') as f:\n",
        "        for d in data:\n",
        "            for i in d.keys():\n",
        "                f.writelines(i + ' ' + d[i] +'\\n')\n",
        "            f.writelines('\\n')"
      ],
      "metadata": {
        "id": "oc2NeiJcv0z1"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# We need to save train_ds, val_ds, and test_ds as txt file. This is the required format expected by flairNLP library.\n",
        "\n",
        "dump_to_txt_file(train_ner_data, \"data/ner/train_ner_data.txt\")\n",
        "dump_to_txt_file(val_ner_data, \"data/ner/val_ner_data.txt\")\n",
        "dump_to_txt_file(test_ner_data, \"data/ner/test_ner_data.txt\")"
      ],
      "metadata": {
        "id": "paWtdN1Lv0xu"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "PM7BvbYbngMK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e1915b0d-8b32-44a1-f085-1073f7fae1d9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:15:14,342 Reading data from data/ner\n",
            "2023-09-24 20:15:14,344 Train: data/ner/train_ner_data.txt\n",
            "2023-09-24 20:15:14,345 Dev: data/ner/val_ner_data.txt\n",
            "2023-09-24 20:15:14,346 Test: data/ner/test_ner_data.txt\n"
          ]
        }
      ],
      "source": [
        "# define columns\n",
        "\n",
        "columns = {0: \"text\", 1: \"ner\"}\n",
        "\n",
        "# this is the folder in which train, validation and test files reside\n",
        "data_folder = \"data/ner\"\n",
        "\n",
        "# init a corpus using column format, data folder and the names of the train, val, and test files\n",
        "corpus: Corpus = ColumnCorpus(data_folder, columns,\n",
        "                              train_file=\"train_ner_data.txt\",\n",
        "                              test_file=\"test_ner_data.txt\",\n",
        "                              dev_file=\"val_ner_data.txt\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(corpus.train))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rjBQTO_VqWxc",
        "outputId": "7e1db807-24ff-46d0-d8b3-6ecacb0d6d8a"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "768\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corpus.train[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dRyrEYXSqWvY",
        "outputId": "d4dcd1b4-cefc-4bbb-b472-e73012e04797"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Sentence[76]: \"Good catch . NY law should apply throughout the Americas and in Japan We used UK Europe Singapore on theory counterparties there would be more comfortable with Uk I 'm not sure which Australians prefer What do you think of following : 16 If are accessing this website from North America , South or terms legal Privacy statement shall governed by laws State New York ; if any other location its English Leonardo Pacheco @ENRON Australia\" → [\"NY\"/LOCATION, \"Americas\"/LOCATION, \"Japan\"/LOCATION, \"Europe\"/LOCATION, \"Singapore\"/LOCATION, \"North America\"/LOCATION, \"South\"/LOCATION, \"Leonardo Pacheco\"/PERSON, \"Australia\"/LOCATION]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(corpus.test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C_N0KS-MCZtM",
        "outputId": "678bb951-885d-45c1-f998-9611f8ccd7ee"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "96\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corpus.test[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R5LlzwXtD0v1",
        "outputId": "2a2d558e-87c1-4998-929a-163708bd9f2e"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Sentence[109]: \"Allright fellas here 's the gameplan . I 'll drive leave at 8am tomorrow morning we can meet my house or Matt it doesn 't really matter to me plan on leaving BR no later than 9am sunday morn Geaux Tigers Regards , Kevin A Boone Accenture Houston Consultant Energy (Octel ) 713 .837 .1638 (Client 281 .848 .1619 (C .306 .7940 (H .864 .4149 .A .Boone @accenture .com Our web address is http : / /www .accenture This message for designated recipient only and may contain privileged confidential information If you have received in error please notify sender immediately delete original Any other use of email by prohibited\" → [\"Matt\"/PERSON, \"Accenture\"/ORG]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(corpus.dev))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sNcVU92qCZqm",
        "outputId": "90ab0801-b1ef-42c7-d9c3-bc443000cc2e"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "96\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corpus.dev[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DOREHIS9D4WC",
        "outputId": "38133148-fd9e-4a76-c6dd-a2a63b7f0735"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Sentence[82]: \"Al : Per my voice mail , I have left a message for Mark to give his input . think you should revise the letter as follows subject 's edits \"Please be advised that effective July 17 2000 MGplc was aquired by Enron group MG London Inc part of Effective October 9 changed its name \"Enron Trading Services Inc\" Also please Financial Status (now ) has not If there are any questions call me at 212 715 5845 .\" Alfred Pennisi @MGUSA\" → [\"Al\"/PERSON, \"Mark\"/PERSON, \"MGplc\"/ORG, \"Enron\"/ORG, \"MG London Inc\"/ORG, \"Alfred Pennisi\"/PERSON]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# make the label dictionary from the corpus\n",
        "\n",
        "label_type = 'ner'\n",
        "label_dict = corpus.make_label_dictionary(label_type=label_type, add_unk=False)\n",
        "print(label_dict)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9t0Rfj90qglA",
        "outputId": "1925cce5-a931-40c2-d61c-7f729a802851"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:15:23,527 Computing label dictionary. Progress:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "768it [00:00, 31300.47it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:15:23,562 Dictionary created for label 'ner' with 4 values: PERSON (seen 1625 times), ORG (seen 924 times), LOCATION (seen 464 times), DATE (seen 442 times)\n",
            "Dictionary with 4 tags: PERSON, ORG, LOCATION, DATE\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from flair.embeddings import TransformerWordEmbeddings\n",
        "from flair.models import SequenceTagger\n",
        "from flair.trainers import ModelTrainer\n",
        "import torch\n",
        "\n",
        "# Initialize fine-tuneable transformer embeddings WITH document context\n",
        "# We'll use a verion of BERT namely 'bert-base-uncased'\n",
        "embeddings = TransformerWordEmbeddings(model='bert-base-uncased',\n",
        "                                       layers=\"-1\",\n",
        "                                       subtoken_pooling=\"first\",\n",
        "                                       fine_tune=True,\n",
        "                                       use_context=True,\n",
        "                                       )\n",
        "\n",
        "tagger = SequenceTagger(hidden_size=256,\n",
        "                        embeddings=embeddings,\n",
        "                        tag_dictionary=label_dict,\n",
        "                        tag_type='ner',\n",
        "                        tag_format='BIO',\n",
        "                        use_crf=False,\n",
        "                        use_rnn=False,\n",
        "                        reproject_embeddings=False,\n",
        "                        )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZMbmUmTOqgi_",
        "outputId": "73e4f797-fca6-42cd-ce09-80f05dad1fe1"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:15:31,866 SequenceTagger predicts: Dictionary with 9 tags: O, B-PERSON, I-PERSON, B-ORG, I-ORG, B-LOCATION, I-LOCATION, B-DATE, I-DATE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "6D9RjmLjngKT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e2452e4-6dc1-4a0a-f638-d0fa859b380a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<flair.trainers.trainer.ModelTrainer at 0x7ca3d922b610>"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "# Initialize trainer\n",
        "\n",
        "trainer = ModelTrainer(tagger, corpus)\n",
        "trainer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "GkiUpKlxngIJ"
      },
      "outputs": [],
      "source": [
        "# Define function to determine the current date\n",
        "\n",
        "from datetime import datetime\n",
        "\n",
        "def get_current_date(date_format=\"%Y%m%d\"):\n",
        "    \"\"\" Get current time\n",
        "\n",
        "    :param date_format: input date format\n",
        "    :return: string date\n",
        "    \"\"\"\n",
        "    now = datetime.now()\n",
        "    current_time = now.strftime(f\"{date_format}\")\n",
        "    return current_time"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The current date will serve as the model version number\n",
        "\n",
        "model_version = get_current_date(date_format=\"%Y%m%d%H%M\")\n",
        "model_version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "PGnkusbMwP4V",
        "outputId": "feb3bf48-1a55-4c8d-fcce-54c41f7d7ed7"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'202309241957'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "LR = 5e-5\n",
        "BATCH_SIZE = 16\n",
        "NUM_EPOCHS = 5\n",
        "\n",
        "trainer.fine_tune(f\"models/taggers/ner/{model_version}\",\n",
        "              learning_rate=LR,\n",
        "              mini_batch_size=BATCH_SIZE,\n",
        "              max_epochs=NUM_EPOCHS,\n",
        "              optimizer=torch.optim.AdamW,\n",
        "              main_evaluation_metric=(\"micro avg\", \"f1-score\")\n",
        "              )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ByzpGWkMwP2t",
        "outputId": "28b1ff17-9716-4ab4-cdca-132a3b2fd8ef"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 19:57:20,101 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:20,112 Model: \"SequenceTagger(\n",
            "  (embeddings): TransformerWordEmbeddings(\n",
            "    (model): BertModel(\n",
            "      (embeddings): BertEmbeddings(\n",
            "        (word_embeddings): Embedding(30523, 768)\n",
            "        (position_embeddings): Embedding(512, 768)\n",
            "        (token_type_embeddings): Embedding(2, 768)\n",
            "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (encoder): BertEncoder(\n",
            "        (layer): ModuleList(\n",
            "          (0-11): 12 x BertLayer(\n",
            "            (attention): BertAttention(\n",
            "              (self): BertSelfAttention(\n",
            "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (dropout): Dropout(p=0.1, inplace=False)\n",
            "              )\n",
            "              (output): BertSelfOutput(\n",
            "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "                (dropout): Dropout(p=0.1, inplace=False)\n",
            "              )\n",
            "            )\n",
            "            (intermediate): BertIntermediate(\n",
            "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (intermediate_act_fn): GELUActivation()\n",
            "            )\n",
            "            (output): BertOutput(\n",
            "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (pooler): BertPooler(\n",
            "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "        (activation): Tanh()\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (locked_dropout): LockedDropout(p=0.5)\n",
            "  (linear): Linear(in_features=768, out_features=9, bias=True)\n",
            "  (loss_function): CrossEntropyLoss()\n",
            ")\"\n",
            "2023-09-24 19:57:20,124 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:20,131 Corpus: \"Corpus: 768 train + 96 dev + 96 test sentences\"\n",
            "2023-09-24 19:57:20,133 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:20,140 Parameters:\n",
            "2023-09-24 19:57:20,142  - learning_rate: \"0.000050\"\n",
            "2023-09-24 19:57:20,146  - mini_batch_size: \"16\"\n",
            "2023-09-24 19:57:20,148  - patience: \"3\"\n",
            "2023-09-24 19:57:20,155  - anneal_factor: \"0.5\"\n",
            "2023-09-24 19:57:20,163  - max_epochs: \"5\"\n",
            "2023-09-24 19:57:20,171  - shuffle: \"True\"\n",
            "2023-09-24 19:57:20,172  - train_with_dev: \"False\"\n",
            "2023-09-24 19:57:20,173  - batch_growth_annealing: \"False\"\n",
            "2023-09-24 19:57:20,181 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:20,185 Model training base path: \"models/taggers/ner/202309241957\"\n",
            "2023-09-24 19:57:20,186 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:20,197 Device: cuda:0\n",
            "2023-09-24 19:57:20,205 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:20,206 Embeddings storage mode: none\n",
            "2023-09-24 19:57:20,207 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:57:28,529 epoch 1 - iter 4/48 - loss 1.92894218 - time (sec): 8.31 - samples/sec: 724.17 - lr: 0.000008\n",
            "2023-09-24 19:57:32,002 epoch 1 - iter 8/48 - loss 1.73692696 - time (sec): 11.79 - samples/sec: 1031.99 - lr: 0.000017\n",
            "2023-09-24 19:57:36,186 epoch 1 - iter 12/48 - loss 1.46935047 - time (sec): 15.97 - samples/sec: 1127.97 - lr: 0.000025\n",
            "2023-09-24 19:57:40,954 epoch 1 - iter 16/48 - loss 1.20005544 - time (sec): 20.74 - samples/sec: 1172.85 - lr: 0.000033\n",
            "2023-09-24 19:57:45,362 epoch 1 - iter 20/48 - loss 1.03635389 - time (sec): 25.15 - samples/sec: 1217.92 - lr: 0.000042\n",
            "2023-09-24 19:57:48,794 epoch 1 - iter 24/48 - loss 0.94012017 - time (sec): 28.58 - samples/sec: 1283.20 - lr: 0.000050\n",
            "2023-09-24 19:57:52,382 epoch 1 - iter 28/48 - loss 0.86059935 - time (sec): 32.17 - samples/sec: 1328.80 - lr: 0.000049\n",
            "2023-09-24 19:57:55,672 epoch 1 - iter 32/48 - loss 0.79847938 - time (sec): 35.46 - samples/sec: 1375.40 - lr: 0.000048\n",
            "2023-09-24 19:57:59,205 epoch 1 - iter 36/48 - loss 0.75076891 - time (sec): 38.99 - samples/sec: 1408.06 - lr: 0.000047\n",
            "2023-09-24 19:58:03,326 epoch 1 - iter 40/48 - loss 0.70048022 - time (sec): 43.11 - samples/sec: 1418.49 - lr: 0.000046\n",
            "2023-09-24 19:58:06,657 epoch 1 - iter 44/48 - loss 0.66296341 - time (sec): 46.44 - samples/sec: 1446.07 - lr: 0.000045\n",
            "2023-09-24 19:58:10,935 epoch 1 - iter 48/48 - loss 0.62929893 - time (sec): 50.72 - samples/sec: 1444.64 - lr: 0.000045\n",
            "2023-09-24 19:58:10,938 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:58:10,940 EPOCH 1 done: loss 0.6293 - lr 0.000045\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 19:58:13,996 Evaluating as a multi-label problem: False\n",
            "2023-09-24 19:58:14,023 DEV : loss 0.2221193164587021 - f1-score (micro avg)  0.2345\n",
            "2023-09-24 19:58:14,036 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 19:58:19,557 epoch 2 - iter 4/48 - loss 0.22381216 - time (sec): 5.52 - samples/sec: 1145.54 - lr: 0.000044\n",
            "2023-09-24 19:58:22,941 epoch 2 - iter 8/48 - loss 0.22171547 - time (sec): 8.90 - samples/sec: 1384.35 - lr: 0.000043\n",
            "2023-09-24 19:58:26,869 epoch 2 - iter 12/48 - loss 0.21810096 - time (sec): 12.83 - samples/sec: 1427.16 - lr: 0.000042\n",
            "2023-09-24 19:58:31,065 epoch 2 - iter 16/48 - loss 0.21696108 - time (sec): 17.02 - samples/sec: 1435.37 - lr: 0.000041\n",
            "2023-09-24 19:58:36,613 epoch 2 - iter 20/48 - loss 0.21389038 - time (sec): 22.57 - samples/sec: 1353.18 - lr: 0.000040\n",
            "2023-09-24 19:58:40,498 epoch 2 - iter 24/48 - loss 0.21148096 - time (sec): 26.46 - samples/sec: 1382.12 - lr: 0.000039\n",
            "2023-09-24 19:58:44,182 epoch 2 - iter 28/48 - loss 0.21307722 - time (sec): 30.14 - samples/sec: 1410.77 - lr: 0.000038\n",
            "2023-09-24 19:58:48,486 epoch 2 - iter 32/48 - loss 0.20685158 - time (sec): 34.45 - samples/sec: 1409.18 - lr: 0.000037\n",
            "2023-09-24 19:58:51,998 epoch 2 - iter 36/48 - loss 0.20391004 - time (sec): 37.96 - samples/sec: 1435.04 - lr: 0.000036\n",
            "2023-09-24 19:58:55,718 epoch 2 - iter 40/48 - loss 0.20554725 - time (sec): 41.68 - samples/sec: 1458.91 - lr: 0.000035\n",
            "2023-09-24 19:58:59,951 epoch 2 - iter 44/48 - loss 0.20164035 - time (sec): 45.91 - samples/sec: 1456.58 - lr: 0.000035\n",
            "2023-09-24 19:59:04,688 epoch 2 - iter 48/48 - loss 0.20286608 - time (sec): 50.65 - samples/sec: 1446.69 - lr: 0.000034\n",
            "2023-09-24 19:59:04,710 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 19:59:04,717 EPOCH 2 done: loss 0.2029 - lr 0.000034\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:04<00:00,  1.40it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 19:59:09,044 Evaluating as a multi-label problem: False\n",
            "2023-09-24 19:59:09,069 DEV : loss 0.1413034051656723 - f1-score (micro avg)  0.6184\n",
            "2023-09-24 19:59:09,081 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 19:59:13,462 epoch 3 - iter 4/48 - loss 0.13167093 - time (sec): 4.38 - samples/sec: 1373.31 - lr: 0.000033\n",
            "2023-09-24 19:59:17,438 epoch 3 - iter 8/48 - loss 0.14176730 - time (sec): 8.36 - samples/sec: 1439.25 - lr: 0.000032\n",
            "2023-09-24 19:59:21,491 epoch 3 - iter 12/48 - loss 0.14270455 - time (sec): 12.41 - samples/sec: 1469.94 - lr: 0.000031\n",
            "2023-09-24 19:59:26,195 epoch 3 - iter 16/48 - loss 0.14008014 - time (sec): 17.11 - samples/sec: 1411.57 - lr: 0.000030\n",
            "2023-09-24 19:59:30,202 epoch 3 - iter 20/48 - loss 0.13761293 - time (sec): 21.12 - samples/sec: 1425.42 - lr: 0.000029\n",
            "2023-09-24 19:59:34,570 epoch 3 - iter 24/48 - loss 0.14276203 - time (sec): 25.49 - samples/sec: 1427.09 - lr: 0.000028\n",
            "2023-09-24 19:59:39,032 epoch 3 - iter 28/48 - loss 0.14102589 - time (sec): 29.95 - samples/sec: 1415.18 - lr: 0.000027\n",
            "2023-09-24 19:59:43,819 epoch 3 - iter 32/48 - loss 0.13952291 - time (sec): 34.74 - samples/sec: 1401.58 - lr: 0.000026\n",
            "2023-09-24 19:59:48,055 epoch 3 - iter 36/48 - loss 0.13868858 - time (sec): 38.97 - samples/sec: 1408.50 - lr: 0.000025\n",
            "2023-09-24 19:59:52,665 epoch 3 - iter 40/48 - loss 0.14064196 - time (sec): 43.58 - samples/sec: 1400.04 - lr: 0.000025\n",
            "2023-09-24 19:59:57,251 epoch 3 - iter 44/48 - loss 0.14199866 - time (sec): 48.17 - samples/sec: 1392.76 - lr: 0.000024\n",
            "2023-09-24 20:00:01,632 epoch 3 - iter 48/48 - loss 0.14068687 - time (sec): 52.55 - samples/sec: 1394.36 - lr: 0.000023\n",
            "2023-09-24 20:00:01,639 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:00:01,642 EPOCH 3 done: loss 0.1407 - lr 0.000023\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.83it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:00:04,939 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:00:04,955 DEV : loss 0.12771132588386536 - f1-score (micro avg)  0.6226\n",
            "2023-09-24 20:00:04,963 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:00:09,790 epoch 4 - iter 4/48 - loss 0.11859390 - time (sec): 4.83 - samples/sec: 1279.75 - lr: 0.000022\n",
            "2023-09-24 20:00:14,475 epoch 4 - iter 8/48 - loss 0.11040869 - time (sec): 9.51 - samples/sec: 1301.33 - lr: 0.000021\n",
            "2023-09-24 20:00:18,754 epoch 4 - iter 12/48 - loss 0.11633330 - time (sec): 13.79 - samples/sec: 1331.16 - lr: 0.000020\n",
            "2023-09-24 20:00:23,273 epoch 4 - iter 16/48 - loss 0.12161531 - time (sec): 18.31 - samples/sec: 1340.03 - lr: 0.000019\n",
            "2023-09-24 20:00:27,322 epoch 4 - iter 20/48 - loss 0.12280147 - time (sec): 22.36 - samples/sec: 1379.69 - lr: 0.000018\n",
            "2023-09-24 20:00:31,283 epoch 4 - iter 24/48 - loss 0.12028609 - time (sec): 26.32 - samples/sec: 1395.49 - lr: 0.000017\n",
            "2023-09-24 20:00:35,604 epoch 4 - iter 28/48 - loss 0.11655327 - time (sec): 30.64 - samples/sec: 1402.66 - lr: 0.000016\n",
            "2023-09-24 20:00:39,147 epoch 4 - iter 32/48 - loss 0.11664830 - time (sec): 34.18 - samples/sec: 1431.57 - lr: 0.000015\n",
            "2023-09-24 20:00:44,263 epoch 4 - iter 36/48 - loss 0.11892099 - time (sec): 39.30 - samples/sec: 1405.52 - lr: 0.000015\n",
            "2023-09-24 20:00:48,531 epoch 4 - iter 40/48 - loss 0.11658320 - time (sec): 43.57 - samples/sec: 1406.53 - lr: 0.000014\n",
            "2023-09-24 20:00:53,051 epoch 4 - iter 44/48 - loss 0.11430632 - time (sec): 48.09 - samples/sec: 1399.22 - lr: 0.000013\n",
            "2023-09-24 20:00:57,405 epoch 4 - iter 48/48 - loss 0.11521556 - time (sec): 52.44 - samples/sec: 1397.25 - lr: 0.000012\n",
            "2023-09-24 20:00:57,410 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:00:57,412 EPOCH 4 done: loss 0.1152 - lr 0.000012\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.96it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:01:00,483 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:01:00,500 DEV : loss 0.1295626312494278 - f1-score (micro avg)  0.6123\n",
            "2023-09-24 20:01:00,511 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:01:04,612 epoch 5 - iter 4/48 - loss 0.10074218 - time (sec): 4.10 - samples/sec: 1428.58 - lr: 0.000011\n",
            "2023-09-24 20:01:08,402 epoch 5 - iter 8/48 - loss 0.09493009 - time (sec): 7.89 - samples/sec: 1542.30 - lr: 0.000010\n",
            "2023-09-24 20:01:12,549 epoch 5 - iter 12/48 - loss 0.09336834 - time (sec): 12.04 - samples/sec: 1522.44 - lr: 0.000009\n",
            "2023-09-24 20:01:17,808 epoch 5 - iter 16/48 - loss 0.09204279 - time (sec): 17.29 - samples/sec: 1412.57 - lr: 0.000008\n",
            "2023-09-24 20:01:21,839 epoch 5 - iter 20/48 - loss 0.09590692 - time (sec): 21.33 - samples/sec: 1437.34 - lr: 0.000007\n",
            "2023-09-24 20:01:26,606 epoch 5 - iter 24/48 - loss 0.09548017 - time (sec): 26.09 - samples/sec: 1407.30 - lr: 0.000006\n",
            "2023-09-24 20:01:32,085 epoch 5 - iter 28/48 - loss 0.09449493 - time (sec): 31.57 - samples/sec: 1360.48 - lr: 0.000005\n",
            "2023-09-24 20:01:36,386 epoch 5 - iter 32/48 - loss 0.09681371 - time (sec): 35.87 - samples/sec: 1368.41 - lr: 0.000005\n",
            "2023-09-24 20:01:40,869 epoch 5 - iter 36/48 - loss 0.09944375 - time (sec): 40.36 - samples/sec: 1363.68 - lr: 0.000004\n",
            "2023-09-24 20:01:44,755 epoch 5 - iter 40/48 - loss 0.10005729 - time (sec): 44.24 - samples/sec: 1381.96 - lr: 0.000003\n",
            "2023-09-24 20:01:48,784 epoch 5 - iter 44/48 - loss 0.09983516 - time (sec): 48.27 - samples/sec: 1389.34 - lr: 0.000002\n",
            "2023-09-24 20:01:53,125 epoch 5 - iter 48/48 - loss 0.09956445 - time (sec): 52.61 - samples/sec: 1392.70 - lr: 0.000001\n",
            "2023-09-24 20:01:53,128 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:01:53,130 EPOCH 5 done: loss 0.0996 - lr 0.000001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.99it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:01:56,167 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:01:56,189 DEV : loss 0.12764804065227509 - f1-score (micro avg)  0.626\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:01:58,170 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:01:58,175 Testing using last state of model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.92it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:02:01,349 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:02:01,372 0.5578\t0.6268\t0.5903\t0.4384\n",
            "2023-09-24 20:02:01,375 \n",
            "Results:\n",
            "- F-score (micro) 0.5903\n",
            "- F-score (macro) 0.5209\n",
            "- Accuracy 0.4384\n",
            "\n",
            "By class:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "      PERSON     0.7269    0.7811    0.7530       201\n",
            "         ORG     0.4754    0.5839    0.5241       149\n",
            "    LOCATION     0.5135    0.5135    0.5135        74\n",
            "        DATE     0.2698    0.3208    0.2931        53\n",
            "\n",
            "   micro avg     0.5578    0.6268    0.5903       477\n",
            "   macro avg     0.4964    0.5498    0.5209       477\n",
            "weighted avg     0.5644    0.6268    0.5932       477\n",
            "\n",
            "2023-09-24 20:02:01,378 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_score': 0.5903257650542941, 'dev_score_history': [0.2345058626465662, 0.618421052631579, 0.6225961538461537, 0.612330198537095, 0.6260330578512396], 'train_loss_history': [0.629298929330472, 0.20286607726690928, 0.14068687148249842, 0.11521556088726312, 0.09956445255606491], 'dev_loss_history': [0.2221193164587021, 0.1413034051656723, 0.12771132588386536, 0.1295626312494278, 0.12764804065227509]}"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "history = defaultdict(list)\n",
        "history['train_loss_history'] = [0.629298929330472, 0.20286607726690928, 0.14068687148249842, 0.11521556088726312, 0.09956445255606491]\n",
        "history['dev_loss_history'] = [0.2221193164587021, 0.1413034051656723, 0.12771132588386536, 0.1295626312494278, 0.12764804065227509]\n",
        "\n",
        "plt.figure()\n",
        "plt.plot(history['train_loss_history'], label='train loss')\n",
        "plt.plot(history['dev_loss_history'], label='val loss')\n",
        "plt.title('Training history')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()\n",
        "plt.ylim([0, 1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "id": "YtjlKkfYyVVF",
        "outputId": "2fdf99de-f941-4f38-b1fa-69fe1129fa4e"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.0, 1.0)"
            ]
          },
          "metadata": {},
          "execution_count": 35
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRqUlEQVR4nO3deXgT1f4G8HeSNknTfaELpVD2nZa1FkRAqoBcBERZRFnUn1cEBCsqqOwqiICIIIhXRb0om4IomwVZFPCCQFnLXmhZ2lJK9zZtk/n9kTZt6N6mnWT6fp4nD8nkTOY7jDEvZ86ZEURRFEFEREQkEwqpCyAiIiKyJIYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiqpJx48YhMDCwSuvOmTMHgiBYtqAK6t27N9q1a1duu+vXr0MQBKxdu7bmiyIii2K4IZIZQRAq9Ni/f7/UpcrS559/zkBEJDGB95Yikpf//ve/Zq+/++47RERE4Pvvvzdb/thjj8HHx6fK28nNzYXBYIBara70unl5ecjLy4NGo6ny9quqd+/eSExMxNmzZ8tsJ4oidDod7O3toVQqK/z57dq1g5eXF8MjkYTspC6AiCzrueeeM3v9999/IyIiotjyB2VmZkKr1VZ4O/b29lWqDwDs7OxgZ2fd//sRBEGS8FWS7OxsqFQqKBTsbCeqCH5TiOqggnEnx48fxyOPPAKtVot33nkHAPDLL79g4MCBqF+/PtRqNZo2bYr58+dDr9ebfcaDY24KxqgsXrwYa9asQdOmTaFWq9G1a1ccO3bMbN2SxtwIgoBJkyZh69ataNeuHdRqNdq2bYtdu3YVq3///v3o0qULNBoNmjZtii+++KLS43jOnz+PPn36QKvVwt/fH4sWLTJ7v6QxN3FxcRg/fjwaNGgAtVoNPz8/DB48GNevXwcABAYG4ty5czhw4IDp9F/v3r1N61+7dg3PPPMMPDw8oNVq8dBDD2H79u3F9k0QBKxfvx7vvfce/P39odVqERkZCUEQ8MknnxTbl8OHD0MQBPz4448V3n8iObPufzoRUY25d+8eBgwYgJEjR+K5554znaJau3YtnJycEB4eDicnJ/zxxx+YNWsWUlNT8fHHH5f7uT/88APS0tLw73//G4IgYNGiRXjqqadw7dq1cnt7/vrrL/z888949dVX4ezsjOXLl2PYsGGIiYmBp6cnAODkyZPo378//Pz8MHfuXOj1esybNw/16tWr8L7fv38f/fv3x1NPPYXhw4dj8+bNePvtt9G+fXsMGDCg1PWGDRuGc+fOYfLkyQgMDERCQgIiIiIQExODwMBALFu2DJMnT4aTkxPeffddADD9vcbHx6N79+7IzMzEa6+9Bk9PT3z77bd48sknsXnzZgwdOtRsW/Pnz4dKpcK0adOg0+nQqlUr9OjRA+vWrcPrr79u1nbdunVwdnbG4MGDK/x3QCRrIhHJ2sSJE8UHv+q9evUSAYirV68u1j4zM7PYsn//+9+iVqsVs7OzTcvGjh0rNmrUyPQ6OjpaBCB6enqKSUlJpuW//PKLCED89ddfTctmz55drCYAokqlEq9cuWJadurUKRGA+Nlnn5mWDRo0SNRqteKtW7dMyy5fviza2dkV+8ySFOz7d999Z1qm0+lEX19fcdiwYcX255tvvhFFURTv378vAhA//vjjMj+/bdu2Yq9evYotnzp1qghA/PPPP03L0tLSxMaNG4uBgYGiXq8XRVEU9+3bJwIQmzRpUuxYfPHFFyIAMSoqyrQsJydH9PLyEseOHVvuvhPVFTwtRVRHqdVqjB8/vthyBwcH0/O0tDQkJiaiZ8+eyMzMxIULF8r93BEjRsDd3d30umfPngCMp2TKExYWhqZNm5ped+jQAS4uLqZ19Xo99uzZgyFDhqB+/fqmds2aNSuzx+VBTk5OZmOQVCoVunXrVmaNDg4OUKlU2L9/P+7fv1/hbRXYsWMHunXrhocfftisjpdffhnXr1/H+fPnzdqPHTvW7FgAwPDhw6HRaLBu3TrTst27dyMxMbHcMVVEdQnDDVEd5e/vD5VKVWz5uXPnMHToULi6usLFxQX16tUz/XCmpKSU+7kNGzY0e10QdCoSCB5ct2D9gnUTEhKQlZWFZs2aFWtX0rLSNGjQoNj4nKLbKYlarcZHH32EnTt3wsfHB4888ggWLVqEuLi4Cm3zxo0baNmyZbHlrVu3Nr1fVOPGjYu1dXNzw6BBg/DDDz+Ylq1btw7+/v549NFHK1QHUV3AcENURz3YKwAAycnJ6NWrF06dOoV58+bh119/RUREBD766CMAgMFgKPdzS5s2LVbgqhPVWbcyqrqdqVOn4tKlS1iwYAE0Gg1mzpyJ1q1b4+TJkxatDyj5+ADAmDFjcO3aNRw+fBhpaWnYtm0bRo0axZlUREVwQDERmezfvx/37t3Dzz//jEceecS0PDo6WsKqCnl7e0Oj0eDKlSvF3itpWU1o2rQp3njjDbzxxhu4fPkygoODsWTJEtP1hUqbsdWoUSNcvHix2PKCU32NGjWq0Pb79++PevXqYd26dQgJCUFmZiaef/75Ku4NkTwx6hORSUGPRtEejJycHHz++edSlWRGqVQiLCwMW7duxe3bt03Lr1y5gp07d9botjMzM5GdnW22rGnTpnB2doZOpzMtc3R0RHJycrH1n3jiCRw9ehRHjhwxLcvIyMCaNWsQGBiINm3aVKgOOzs7jBo1Chs3bsTatWvRvn17dOjQoWo7RSRT7LkhIpPu3bvD3d0dY8eOxWuvvQZBEPD9999b/LRQdcyZMwe///47evTogQkTJkCv12PFihVo164dIiMja2y7ly5dQt++fTF8+HC0adMGdnZ22LJlC+Lj4zFy5EhTu86dO2PVqlV4//330axZM3h7e+PRRx/F9OnT8eOPP2LAgAF47bXX4OHhgW+//RbR0dH46aefKnVaacyYMVi+fDn27dtnOmVIRIUYbojIxNPTE7/99hveeOMNvPfee3B3d8dzzz2Hvn37ol+/flKXB8AYHnbu3Ilp06Zh5syZCAgIwLx58xAVFVWh2VxVFRAQgFGjRmHv3r34/vvvYWdnh1atWmHjxo0YNmyYqd2sWbNw48YNLFq0CGlpaejVqxceffRR+Pj44PDhw3j77bfx2WefITs7Gx06dMCvv/6KgQMHVqqWzp07o23btoiKisLo0aMtvatENo/3liIiWRgyZAjOnTuHy5cvS11KrejYsSM8PDywd+9eqUshsjocc0NENicrK8vs9eXLl7Fjxw6zWx3I2T///IPIyEiMGTNG6lKIrBJ7bojI5vj5+WHcuHFo0qQJbty4gVWrVkGn0+HkyZNo3ry51OXVmLNnz+L48eNYsmQJEhMTce3aNau5uSeRNeGYGyKyOf3798ePP/6IuLg4qNVqhIaG4sMPP5R1sAGAzZs3Y968eWjZsiV+/PFHBhuiUkjac3Pw4EF8/PHHOH78OO7cuYMtW7ZgyJAhZa6zf/9+hIeH49y5cwgICMB7772HcePG1Uq9REREZP0kHXOTkZGBoKAgrFy5skLto6OjMXDgQPTp0weRkZGYOnUqXnrpJezevbuGKyUiIiJbYTVjbgRBKLfn5u2338b27dtx9uxZ07KRI0ciOTkZu3btqoUqiYiIyNrZ1JibI0eOICwszGxZv379MHXq1FLX0el0ZlcPNRgMSEpKgqenZ6mXSSciIiLrIooi0tLSUL9+/XIvemlT4SYuLg4+Pj5my3x8fJCamoqsrKwSbzS3YMECzJ07t7ZKJCIiohoUGxuLBg0alNnGpsJNVcyYMQPh4eGm1ykpKWjYsCFiY2Ph4uIiYWVERERUUampqQgICICzs3O5bW0q3Pj6+iI+Pt5sWXx8PFxcXErstQEAtVoNtVpdbLmLiwvDDRERkY2pyJASm7pCcWhoaLFLjUdERCA0NFSiioiIiMjaSBpu0tPTERkZabqTb3R0NCIjIxETEwPAeEqp6OXFX3nlFVy7dg1vvfUWLly4gM8//xwbN27E66+/LkX5REREZIUkDTf//PMPOnbsiI4dOwIAwsPD0bFjR8yaNQsAcOfOHVPQAYDGjRtj+/btiIiIQFBQEJYsWYL//Oc/VnO3YiIiIpKe1VznprakpqbC1dUVKSkpHHNDRCRTer0eubm5UpdBlaRSqUqd5l2Z32+bGlBMRERUFlEUERcXh+TkZKlLoSpQKBRo3LgxVCpVtT6H4YaIiGSjINh4e3tDq9XyYq02xGAw4Pbt27hz5w4aNmxYrWPHcENERLKg1+tNwcbT01PqcqgK6tWrh9u3byMvLw/29vZV/hybmgpORERUmoIxNlqtVuJKqKoKTkfp9fpqfQ7DDRERyQpPRdkuSx07hhsiIiKSFYYbIiIiGQkMDMSyZcsk/wwpcUAxERGRhHr37o3g4GCLhYljx47B0dHRIp9lqxhuiIiIrJwoitDr9bCzK/9nu169erVQkXXjaSkiIiKJjBs3DgcOHMCnn34KQRAgCAKuX7+O/fv3QxAE7Ny5E507d4ZarcZff/2Fq1evYvDgwfDx8YGTkxO6du2KPXv2mH3mg6eUBEHAf/7zHwwdOhRarRbNmzfHtm3bKlVnTEwMBg8eDCcnJ7i4uGD48OGIj483vX/q1Cn06dMHzs7OcHFxQefOnfHPP/8AAG7cuIFBgwbB3d0djo6OaNu2LXbs2FH1v7QKYM8NERHJkiiKyMqt3pTiqnKwV1Zo5s+nn36KS5cuoV27dpg3bx4AY8/L9evXAQDTp0/H4sWL0aRJE7i7uyM2NhZPPPEEPvjgA6jVanz33XcYNGgQLl68iIYNG5a6nblz52LRokX4+OOP8dlnn2H06NG4ceMGPDw8yq3RYDCYgs2BAweQl5eHiRMnYsSIEdi/fz8AYPTo0ejYsSNWrVoFpVKJyMhI03VqJk6ciJycHBw8eBCOjo44f/48nJycyt1udTDcEBGRLGXl6tFm1m5Jtn1+Xj9oVeX/xLq6ukKlUkGr1cLX17fY+/PmzcNjjz1meu3h4YGgoCDT6/nz52PLli3Ytm0bJk2aVOp2xo0bh1GjRgEAPvzwQyxfvhxHjx5F//79y61x7969OHPmDKKjoxEQEAAA+O6779C2bVscO3YMXbt2RUxMDN588020atUKANC8eXPT+jExMRg2bBjat28PAGjSpEm526wunpYiIiKyUl26dDF7nZ6ejmnTpqF169Zwc3ODk5MToqKiEBMTU+bndOjQwfTc0dERLi4uSEhIqFANUVFRCAgIMAUbAGjTpg3c3NwQFRUFAAgPD8dLL72EsLAwLFy4EFevXjW1fe211/D++++jR48emD17Nk6fPl2h7VYHe26IiEiWHOyVOD+vn2TbtoQHZz1NmzYNERERWLx4MZo1awYHBwc8/fTTyMnJKfNzHryVgSAIMBgMFqkRAObMmYNnn30W27dvx86dOzF79mysX78eQ4cOxUsvvYR+/fph+/bt+P3337FgwQIsWbIEkydPttj2H8RwQ0REsiQIQoVODUlNpVJV+HYDhw4dwrhx4zB06FAAxp6cgvE5NaV169aIjY1FbGysqffm/PnzSE5ORps2bUztWrRogRYtWuD111/HqFGj8M0335jqDAgIwCuvvIJXXnkFM2bMwJdfflmj4YanpYiIiCQUGBiI//3vf7h+/ToSExPL7FFp3rw5fv75Z0RGRuLUqVN49tlnLdoDU5KwsDC0b98eo0ePxokTJ3D06FGMGTMGvXr1QpcuXZCVlYVJkyZh//79uHHjBg4dOoRjx46hdevWAICpU6di9+7diI6OxokTJ7Bv3z7TezWF4YaIiEhC06ZNg1KpRJs2bVCvXr0yx88sXboU7u7u6N69OwYNGoR+/fqhU6dONVqfIAj45Zdf4O7ujkceeQRhYWFo0qQJNmzYAABQKpW4d+8exowZgxYtWmD48OEYMGAA5s6dC8B4E8yJEyeidevW6N+/P1q0aIHPP/+8ZmsWRVGs0S1YmdTUVLi6uiIlJQUuLi5Sl0NERBaSnZ2N6OhoNG7cGBqNRupyqArKOoaV+f1mzw0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0REZGNCwwMxLJly0p9f9y4cRgyZEit1SM1hhsiIiKSFYYbIiIikhWGGyIiIomsWbMG9evXh8FgMFs+ePBgvPDCCwCAq1evYvDgwfDx8YGTkxO6du2KPXv2VGu7Op0Or732Gry9vaHRaPDwww/j2LFjpvfv37+P0aNHo169enBwcEDz5s3xzTffAABycnIwadIk+Pn5QaPRoFGjRliwYEG16rE0O6kLICIiqhGiCORmSrNtey0gCOU2e+aZZzB58mTs27cPffv2BQAkJSVh165d2LFjBwAgPT0dTzzxBD744AOo1Wp89913GDRoEC5evIiGDRtWqby33noLP/30E7799ls0atQIixYtQr9+/XDlyhV4eHhg5syZOH/+PHbu3AkvLy9cuXIFWVlZAIDly5dj27Zt2LhxIxo2bIjY2FjExsZWqY6awnBDRETylJsJfFhfmm2/cxtQOZbbzN3dHQMGDMAPP/xgCjebN2+Gl5cX+vTpAwAICgpCUFCQaZ358+djy5Yt2LZtGyZNmlTp0jIyMrBq1SqsXbsWAwYMAAB8+eWXiIiIwFdffYU333wTMTEx6NixI7p06QLAOGC5QExMDJo3b46HH34YgiCgUaNGla6hpvG0FBERkYRGjx6Nn376CTqdDgCwbt06jBw5EgqF8Sc6PT0d06ZNQ+vWreHm5gYnJydERUUhJiamStu7evUqcnNz0aNHD9Mye3t7dOvWDVFRUQCACRMmYP369QgODsZbb72Fw4cPm9qOGzcOkZGRaNmyJV577TX8/vvvVd31GsOeGyIikid7rbEHRaptV9CgQYMgiiK2b9+Orl274s8//8Qnn3xien/atGmIiIjA4sWL0axZMzg4OODpp59GTk5OTVQOABgwYABu3LiBHTt2ICIiAn379sXEiROxePFidOrUCdHR0di5cyf27NmD4cOHIywsDJs3b66xeiqL4YaIiORJECp0akhqGo0GTz31FNatW4crV66gZcuW6NSpk+n9Q4cOYdy4cRg6dCgAY0/O9evXq7y9pk2bQqVS4dChQ6ZTSrm5uTh27BimTp1qalevXj2MHTsWY8eORc+ePfHmm29i8eLFAAAXFxeMGDECI0aMwNNPP43+/fsjKSkJHh4eVa7LkhhuiIiIJDZ69Gj861//wrlz5/Dcc8+Zvde8eXP8/PPPGDRoEARBwMyZM4vNrqoMR0dHTJgwAW+++SY8PDzQsGFDLFq0CJmZmXjxxRcBALNmzULnzp3Rtm1b6HQ6/Pbbb2jdujUAYOnSpfDz80PHjh2hUCiwadMm+Pr6ws3Nrco1WRrDDRERkcQeffRReHh44OLFi3j22WfN3lu6dCleeOEFdO/eHV5eXnj77beRmppare0tXLgQBoMBzz//PNLS0tClSxfs3r0b7u7uAACVSoUZM2bg+vXrcHBwQM+ePbF+/XoAgLOzMxYtWoTLly9DqVSia9eu2LFjh2mMkDUQRFEUpS6iNqWmpsLV1RUpKSlwcXGRuhwiIrKQ7OxsREdHo3HjxtBoNFKXQ1VQ1jGszO+39cQsIiIiIgtguCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIlmpY/NkZMVSx47hhoiIZMHe3h4AkJkp0c0yqdoKrrqsVCqr9Tm8zg0REcmCUqmEm5sbEhISAABarRZCBe7MTdbBYDDg7t270Gq1sLOrXjxhuCEiItnw9fUFAFPAIduiUCjQsGHDaodShhsiIpINQRDg5+cHb29v5ObmSl0OVZJKpbLIlY4ZboiISHaUSmW1x22Q7eKAYiIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFcnDzcqVKxEYGAiNRoOQkBAcPXq0zPbLli1Dy5Yt4eDggICAALz++uvIzs6upWqJiIjI2kkabjZs2IDw8HDMnj0bJ06cQFBQEPr164eEhIQS2//www+YPn06Zs+ejaioKHz11VfYsGED3nnnnVqunIiIiKyVpOFm6dKl+L//+z+MHz8ebdq0werVq6HVavH111+X2P7w4cPo0aMHnn32WQQGBuLxxx/HqFGjyu3tISIiorpDsnCTk5OD48ePIywsrLAYhQJhYWE4cuRIiet0794dx48fN4WZa9euYceOHXjiiSdK3Y5Op0NqaqrZg4iIiOTLTqoNJyYmQq/Xw8fHx2y5j48PLly4UOI6zz77LBITE/Hwww9DFEXk5eXhlVdeKfO01IIFCzB37lyL1k5ERETWS/IBxZWxf/9+fPjhh/j8889x4sQJ/Pzzz9i+fTvmz59f6jozZsxASkqK6REbG1uLFRMREVFtk6znxsvLC0qlEvHx8WbL4+Pj4evrW+I6M2fOxPPPP4+XXnoJANC+fXtkZGTg5ZdfxrvvvguFonhWU6vVUKvVlt8BIiIiskqS9dyoVCp07twZe/fuNS0zGAzYu3cvQkNDS1wnMzOzWIBRKpUAAFEUa65YIiIishmS9dwAQHh4OMaOHYsuXbqgW7duWLZsGTIyMjB+/HgAwJgxY+Dv748FCxYAAAYNGoSlS5eiY8eOCAkJwZUrVzBz5kwMGjTIFHKIiIiobpM03IwYMQJ3797FrFmzEBcXh+DgYOzatcs0yDgmJsasp+a9996DIAh47733cOvWLdSrVw+DBg3CBx98INUuEBERkZURxDp2Pic1NRWurq5ISUmBi4uL1OUQERFRBVTm99umZksRERERlYfhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheHGgtKyc5GanSt1GURERHUaw42FnIpNxhPL/8S7W85CFEWpyyEiIqqzGG4sxCCKuJ2cjV9P3cbPJ25JXQ4REVGdxXBjIR0buuP1sOYAgFm/nMWNexkSV0RERFQ3MdxY0ITezdCtsQcycvSYsj4SuXqD1CURERHVOQw3FqRUCPhkRDBcNHaIjE3Gp3suS10SERFRncNwY2H+bg748Kn2AICV+6/gf9fuSVwRERFR3cJwUwP+1aE+nu7cAKIIvL4hEimZnB5ORERUWxhuasicJ9uikacWt1Oy8c7WM5weTkREVEskDzcrV65EYGAgNBoNQkJCcPTo0TLbJycnY+LEifDz84NarUaLFi2wY8eOWqq24pzUdvh0ZEfYKQRsP30Hm4/flLokIiKiOkHScLNhwwaEh4dj9uzZOHHiBIKCgtCvXz8kJCSU2D4nJwePPfYYrl+/js2bN+PixYv48ssv4e/vX8uVV0xwgBtef6wFAGD2tnO4nsjp4URERDVNECU8XxISEoKuXbtixYoVAACDwYCAgABMnjwZ06dPL9Z+9erV+Pjjj3HhwgXY29tXaZupqalwdXVFSkoKXFxcqlV/RegNIp798m/8LzoJQQ1csXlCd9grJe8wIyIisimV+f2W7Fc2JycHx48fR1hYWGExCgXCwsJw5MiREtfZtm0bQkNDMXHiRPj4+KBdu3b48MMPodfrS92OTqdDamqq2aM2FZ0efupmCpbtuVSr2yciIqprJAs3iYmJ0Ov18PHxMVvu4+ODuLi4Ete5du0aNm/eDL1ejx07dmDmzJlYsmQJ3n///VK3s2DBAri6upoeAQEBFt2Piqjv5oCFwzoAAD7ffxV/c3o4ERFRjbGp8yMGgwHe3t5Ys2YNOnfujBEjRuDdd9/F6tWrS11nxowZSElJMT1iY2NrseJCT7T3w/AunB5ORERU0yQLN15eXlAqlYiPjzdbHh8fD19f3xLX8fPzQ4sWLaBUKk3LWrdujbi4OOTk5JS4jlqthouLi9lDKrMHtUVjL0fcScnGjC2nOT2ciIioBkgWblQqFTp37oy9e/ealhkMBuzduxehoaElrtOjRw9cuXIFBkPhPZsuXboEPz8/qFSqGq+5uhzVdlg2Ihh2CgE7zsRh0z+cHk5ERGRpkp6WCg8Px5dffolvv/0WUVFRmDBhAjIyMjB+/HgAwJgxYzBjxgxT+wkTJiApKQlTpkzBpUuXsH37dnz44YeYOHGiVLtQaUEBbgh/3Dg9fM6v5xDN6eFEREQWZSflxkeMGIG7d+9i1qxZiIuLQ3BwMHbt2mUaZBwTEwOFojB/BQQEYPfu3Xj99dfRoUMH+Pv7Y8qUKXj77bel2oUq+fcjTXHw0l38fS0JU9afxOZXukNlZ1PDn4iIiKyWpNe5kUJtX+emNHdSstB/2Z9IycrFhN5N8Xb/VpLVQkREZO1s4jo3dZ2fqwMW5t89fPWBqzh8NVHiioiIiOSB4UZCA9r7YUSXAIgiEL7hFJIzS57xRURERBXHcCOxWYPaoImXI+JSszHjZ949nIiIqLoYbiTmmH/3cHulgJ1n47DxH2kuMkhERCQXDDdWoH0DV7zxeEsAwJxt53HtbrrEFREREdkuhhsr8XLPJuje1BNZuXpMWR+JnDxD+SsRERFRMQw3VkKhELB0eDDctPY4cysFSyIuSl0SERGRTWK4sSK+rhosfMp49/A1B6/h8BVODyciIqoshhsr07+dL0Z1M04Pf31jJO5ncHo4ERFRZTDcWKGZ/2qDJvUcEZ+qw/SfefdwIiKiymC4sUJalR2W508P330uHuuPcXo4ERFRRTHcWKl2/q6Ylj89fN6v53GV08OJiIgqhOHGiv1fzybo0axgevhJTg8nIiKqAIYbK1YwPdxda4+zt1Kx5HdODyciIioPw42V83HR4KNhxunhXxy8hkOcHk5ERFQmhhsb8HhbXzwb0hAAEM7p4URERGViuLERMwe2QdP86eFv/8Tp4URERKVhuLERDiql6e7hv5+Pxw9HY6QuiYiIyCpVKdzExsbi5s2bptdHjx7F1KlTsWbNGosVRsW183fFW/1aAQDm/3YeVxLSJK6IiIjI+lQp3Dz77LPYt28fACAuLg6PPfYYjh49infffRfz5s2zaIFk7sWHG6Nncy9k5xrw2o+R0OXppS6JiIjIqlQp3Jw9exbdunUDAGzcuBHt2rXD4cOHsW7dOqxdu9aS9dEDFAoBi58JgrvWHufvpGLxbk4PJyIiKqpK4SY3NxdqtRoAsGfPHjz55JMAgFatWuHOnTuWq45K5OOiwaKngwAAX/4ZjT8v35W4IiIiIutRpXDTtm1brF69Gn/++SciIiLQv39/AMDt27fh6elp0QKpZI+18cFzDxmnh7+x8RSSOD2ciIgIQBXDzUcffYQvvvgCvXv3xqhRoxAUZOxF2LZtm+l0FdW8d59og2beTkhI0+GtzZweTkREBACCWMVfRL1ej9TUVLi7u5uWXb9+HVqtFt7e3hYr0NJSU1Ph6uqKlJQUuLi4SF1OtZ2/nYohKw8hR2/A+0Pa4bmHGkldEhERkcVV5ve7Sj03WVlZ0Ol0pmBz48YNLFu2DBcvXrTqYCNHbeq74K3+xruHv7+d08OJiIiqFG4GDx6M7777DgCQnJyMkJAQLFmyBEOGDMGqVassWiCV74UehdPDJ3N6OBER1XFVCjcnTpxAz549AQCbN2+Gj48Pbty4ge+++w7Lly+3aIFUPoVCwJJnguDhqELUnVQs2sXp4UREVHdVKdxkZmbC2dkZAPD777/jqaeegkKhwEMPPYQbN25YtECqGG8XDT5+2nj38K/+isbBS5weTkREdVOVwk2zZs2wdetWxMbGYvfu3Xj88ccBAAkJCbIYpGur+rb2wfP5A4rf2HQK99J1EldERERU+6oUbmbNmoVp06YhMDAQ3bp1Q2hoKABjL07Hjh0tWiBVzrsDW6O5txPucno4ERHVUVWeCh4XF4c7d+4gKCgICoUxIx09ehQuLi5o1aqVRYu0JLlNBS9J1J1UDF5hnB4+f3BbPB8aKHVJRERE1VKZ3+8qh5sCBXcHb9CgQXU+ptbUhXADGMfdzP/tPNR2Cvw6+WG08HGWuiQiIqIqq/Hr3BgMBsybNw+urq5o1KgRGjVqBDc3N8yfPx8Gg6FKRZNlje8eiF4t6kGXZ8BrP55Edi6nhxMRUd1QpXDz7rvvYsWKFVi4cCFOnjyJkydP4sMPP8Rnn32GmTNnWrpGqoKCu4d7OqpwIS6N08OJiKjOqNJpqfr162P16tWmu4EX+OWXX/Dqq6/i1q1bFivQ0urKaakCf1yIxwtr/wEArB3fFb1b8grSRERke2r8tFRSUlKJg4ZbtWqFpKSkqnwk1ZBHW/lgbKhxevi0TaeRyOnhREQkc1UKN0FBQVixYkWx5StWrECHDh2qXRRZ1ownWqOljzMS03V4c9MpTg8nIiJZs6vKSosWLcLAgQOxZ88e0zVujhw5gtjYWOzYscOiBVL1aeyV+HRUMJ5ccQj7Lt7Fd0duYGz3QKnLIiIiqhFV6rnp1asXLl26hKFDhyI5ORnJycl46qmncO7cOXz//feWrpEsoJWvC2YMMJ5K/GBHFC7G8e7hREQkT9W+zk1Rp06dQqdOnaDXW++047o2oLgoURQxfu0x7L94Fy19nPHLpB7Q2CulLouIiKhcNT6gmGyTIAj4+OkgeDmpcDE+DQt3XpC6JCIiIotjuKlj6jmr8fHTQQCAtYevY9/FBIkrIiIisiyGmzqoTytvjMsfUPzmplO4m8bp4UREJB+Vmi311FNPlfl+cnJydWqhWjR9QCv8fe0eLsSl4a3Np/D1uK4QBEHqsoiIiKqtUj03rq6uZT4aNWqEMWPG1FStZEEaeyU+HdkRKjsF9l28i28PX5e6JCIiIouw6GwpW1CXZ0uV5NvD1zF72zmo7BTYNqkHWvny74SIiKwPZ0tRhY0JbYRHW3kjJ8+AKT9G8u7hRERk8xhu6jhBELDo6Q7wclLjYnwaFuyIkrokIiKiamG4IXg5qbH4GeM9wb49cgN/XIiXuCIiIqKqY7ghAEDvlt4Y3yMQAPDmptNISMuWtiAiIqIqYrghk7f7t0IrX2fcy8jBm5tOw2CoU2PNiYhIJhhuyERjr8TyUR2htlPgwKW7WMvp4UREZIMYbshMCx9nvDewNQBg4c4LiLqTKnFFRERElcNwQ8U891AjhLX2Ro7egNd+PMnp4UREZFMYbqgYQRDw0bAOqOesxuWEdHzI6eFERGRDGG6oRJ5Oaix5xnj38O+O3MDeKE4PJyIi28BwQ6V6pEU9vPhwYwDAm5tPIyGV08OJiMj6MdxQmd7q3xKt/VyQlJGDNzad4vRwIiKyegw3VCa1nRLLRwZDbafAn5cT8fWhaKlLIiIiKhPDDZWruY8z3vtXGwDAol0Xce52isQVERERlY7hhirkuZCGCGvtgxy9AVPWRyIrh9PDiYjIOjHcUIUYp4e3Rz1nNa4kpOODHeelLomIiKhEVhFuVq5cicDAQGg0GoSEhODo0aMVWm/9+vUQBAFDhgyp2QIJgHF6+NLhxunh//07BhHnOT2ciIisj+ThZsOGDQgPD8fs2bNx4sQJBAUFoV+/fkhISChzvevXr2PatGno2bNnLVVKANCzeT38X0/j9PC3f+L0cCIisj6Sh5ulS5fi//7v/zB+/Hi0adMGq1evhlarxddff13qOnq9HqNHj8bcuXPRpEmTWqyWAGBav5Zow+nhRERkpSQNNzk5OTh+/DjCwsJMyxQKBcLCwnDkyJFS15s3bx68vb3x4osvlrsNnU6H1NRUswdVj9rOePdwjT2nhxMRkfWRNNwkJiZCr9fDx8fHbLmPjw/i4uJKXOevv/7CV199hS+//LJC21iwYAFcXV1Nj4CAgGrXTUAzbyfM5PRwIiKyQpKflqqMtLQ0PP/88/jyyy/h5eVVoXVmzJiBlJQU0yM2NraGq6w7nu3WEI+18THdPZzTw4mIyBrYSblxLy8vKJVKxMebz7qJj4+Hr69vsfZXr17F9evXMWjQINMyg8EAALCzs8PFixfRtGlTs3XUajXUanUNVE8Fdw8/FXsQV+9mYP728/hwaHupyyIiojpO0p4blUqFzp07Y+/evaZlBoMBe/fuRWhoaLH2rVq1wpkzZxAZGWl6PPnkk+jTpw8iIyN5ykkCHo4qLB0eDAD44X8x2H2u5NOJREREtUXSnhsACA8Px9ixY9GlSxd069YNy5YtQ0ZGBsaPHw8AGDNmDPz9/bFgwQJoNBq0a9fObH03NzcAKLacas/Dzb3w8iNNsObgNUz/6TSCA9zg46KRuiwiIqqjJA83I0aMwN27dzFr1izExcUhODgYu3btMg0yjomJgUJhU0OD6qRpj7fEoSuJOHc7FeEbI/H9CyFQKASpyyIiojpIEEWxTl2kJDU1Fa6urkhJSYGLi4vU5cjKlYR0/OuzP5Gda8A7T7TCy480LX8lIiKiCqjM7ze7RMhimnk7YfagtgCAj3dfxNlbnB5ORES1j+GGLGpk1wD0a+uDXL2I19afRGZOntQlERFRHcNwQxYlCAIWPtUBvi4aXLubgfm/RUldEhER1TEMN2Rx7o4qLB0eBEEAfjwag11nOT2ciIhqD8MN1YjuzYzTwwFg+s+nEZfCu4cTEVHtYLihGvPGYy3R3t8VyZm5CN8YybuHExFRrWC4oRqjslNg2chgONgrcfjqPaz585rUJRERUR3AcEM1qmk9J8weZLx7+OLdF3HmJqeHExFRzWK4oRo3omsA+rf1RZ5BxBRODyciohrGcEM1ThAELBzW3jg9PDED8349L3VJREQkYww3VCvctCosHWGcHr7+WCx2nrkjdUlERCRTDDdUa7o39cIrvYz3m5r+8xncScmSuCIiIpIjhhuqVa+HtUCHBq5IycpF+IZT0HN6OBERWRjDDdUqlZ0Cn47sCK1KiSPX7mHNQU4PJyIiy2K4oVrX2MsRc/LvHr7k94s4fTNZ2oKIiEhWGG5IEs90aYAn2hdMD49Eho7Tw4mIyDIYbkgSgiBgwdAO8HPVIDoxA3N/PSd1SUREJBMMNyQZV609PhkRDEEANv5zEzs4PZyIiCyA4YYk9VATT0womB7+02ncTub0cCIiqh6GG5Lc64+1QFADV6Rm5+H1DZGcHk5ERNXCcEOSs1cWTg//X3QSVh+4KnVJRERkwxhuyCoEejli7pPG6eGfRFzCqdhkaQsiIiKbxXBDVuPpzg0wsIOf6e7hnB5ORERVwXBDVkMQBHw4pD3qu2pw/V4m5mzj9HAiIqo8hhuyKgXTwxUCsOn4TWw/zenhRERUOQw3ZHVCmnji1d7NAAAzfj6NW5weTkRElcBwQ1ZpSlhzBAe4GaeHr+f0cCIiqjiGG7JKxunhwXBUKXH0ehJW7b8idUlERGQjGG7IajXydMTcwe0AAJ/suYyTMfclroiIiGwBww1ZtWGd/PGvDn7Q5989PJ3Tw4mIqBwMN2TVBEHAB0Pbw9/NATFJmZj9C6eHExFR2RhuyOq5OhROD//pxE38euq21CUREZEVY7ghm9CtsQcm9TFOD39nyxncvJ8pcUVERGStGG7IZrzWtzk6NnRDWnYewjec4vRwIiIqEcMN2Qw7pQKfjugIJ7Udjl5Pwuf7OD2ciIiKY7ghm9LQU4t5g413D1+29zJOcHo4ERE9gOGGbM7Qjv54Mqg+9AYRU9dHIi07V+qSiIjIijDckM0RBAHvD23H6eFERFQihhuySS4ae3w60jg9/OeTt/BL5C2pSyIiIivBcEM2q0ugByY92hwA8N6Ws4hN4vRwIiJiuCEb99qjzdCpoRvSdHl4fUMk8vQGqUsiIiKJMdyQTbNTKvDpSOP08H9u3MfKfVelLomIiCTGcEM2L8BDi/eHGO8evvyPyzh+g9PDiYjqMoYbkoUhHf0xJDh/eviGk5weTkRUhzHckGzMG9IODdwdEJuUhVmcHk5EVGcx3JBsFEwPVyoEbOH0cCKiOovhhmSlcyMPTH7UePdwTg8nIqqbGG5Idib1aYYujdyRpsvDlPUnOT2ciKiOYbgh2bFTKvDJiGA4q+1wIiYZn/3Bu4cTEdUlDDckSwEeWrw/1Dg9/LM/LuOf60kSV0RERLWF4YZka3CwP4Z29IdBBKasj0Qqp4cTEdUJDDcka/MGt0WAhwNuJWdh5tazUpdDRES1gOGGZM1ZY49lIzpCqRDwS+RtbDl5U+qSiIiohjHckOx1buSOKX2Ndw+fufUcYu5xejgRkZwx3FCdMLFPM3QNdEe6Lg9TN3B6OBGRnDHcUJ2gVAjG6eEa4/Tw5ZweTkQkWww3VGc0cNfig6HtAQAr/riMY5weTkQkSww3lqLPA/bMAa7sBXI4psNaPRlUH091Mk4Pn7o+EilZnB5ORCQ3DDeWcvsE8NcnwH+fAj5qBKz9F3DwY+DmP8bgQ1Zj3uB2aOihxa3kLLy39SxEUZS6JCIisiCGG0tROQLBzwEuDQB9DnD9T+CP94H/9AUWNQHWjwaOfgncvQTwx1RSTmo7093Dfz11Gz+f4N3DiYjkRBDr2D9bU1NT4erqipSUFLi4uFh+A6II3LsKXNsHRB8Aog8C2SnmbZzrA0165z96Ac6+lq+DyvXZ3stYEnEJjioldkzpiUaejlKXREREpajM7zfDTU0z6IE7kcC1/cZHzN/Gnp2i6rUuDDqNegCaWqiLoDeIGLXmbxy9noTgADdseiUU9kp2ZhIRWSOGmzLUerh5UG6WMeAUhJ07pwAUOQSCEmjQpbBnx78LYKeq/TrriFvJWei/7CDSsvMw+dFmeOPxllKXREREJajM77dV/DN15cqVCAwMhEajQUhICI4ePVpq2y+//BI9e/aEu7s73N3dERYWVmZ7q2PvADTtAzw2F/j3AeCta8Az3wKdxwPujQFRD8T+DzjwEfDNAOCjQOC/TwOHVwBxZwEDLz5nSf5uDljwlHF6+Mp9V3A0mtPDiYhsneQ9Nxs2bMCYMWOwevVqhISEYNmyZdi0aRMuXrwIb2/vYu1Hjx6NHj16oHv37tBoNPjoo4+wZcsWnDt3Dv7+/uVuT/Kem/Lcv2Ecq3NtP3DtAJCZaP6+Yz2gca/C01huDaWoUnambTqFzcdvor6rBuv+7yEEemohCILUZRERUT6bOi0VEhKCrl27YsWKFQAAg8GAgIAATJ48GdOnTy93fb1eD3d3d6xYsQJjxowpt73Vh5uiDAYg4XzhKawbh4DcB66h49Gk8BRWYE9A61H7dcpAui4PA5f/iRv5951y19qjfQM3dPB3RfsGrujQwBW+LhoGHiIiiVTm99uulmoqUU5ODo4fP44ZM2aYlikUCoSFheHIkSMV+ozMzEzk5ubCw6PkH3WdTgedTmd6nZqaWr2ia5NCAfi2Mz66TwLycoCbxwrDzq3jQNI14+OfrwEIgF9QYdhp+JDxNBiVy0lthy+e74x3t5zF6ZvJuJ+Zi4OX7uLgpbumNl5OanTIDzodGriivb8b6jmrJayaiIhKImnPze3bt+Hv74/Dhw8jNDTUtPytt97CgQMH8L///a/cz3j11Vexe/dunDt3DhqNptj7c+bMwdy5c4stt4mem/Jkpxp7cwrCzt0L5u8r1caAU3AKyy8YUChrv04bo8vT41JcOk7fSsbp2BScvpWCS/Fp0BuKf1X8XDX5YccN7f1d0d7fFe6OHABORGRpNnNaqrrhZuHChVi0aBH279+PDh06lNimpJ6bgIAAeYSbB6XeyR+vc8B4nZ20O+bva1yBxo/kh50+xlNaPM1SIdm5epy/k4ozN1Nw6mYyztxMwZW76SVejzHAwwEd/N2MvTsNXNHO3xUuGvvaL5qISEZs5rSUl5cXlEol4uPjzZbHx8fD17fsC9stXrwYCxcuxJ49e0oNNgCgVquhVteRUwcufkDQSONDFIHEy4W9Otf/NF5MMOpX4wMAXAOMPTqNexv/dCo+gJuMNPZKdGrojk4N3U3LMnR5OHc7FadvJuP0zRScuZWC6MQMxCZlITYpC9vPFIbLJl6O+WN3jKGnbX0XaFWSfv2IiGTLKgYUd+vWDZ999hkA44Dihg0bYtKkSaUOKF60aBE++OAD7N69Gw899FCltmdTA4otSZ+XfzHBfcaenZi/AcMDN430bls4XqdRd0DtJEGhti0lKxfnbhlPZRWEnpv3s4q1UwhAM28ntPd3Q1CA8XRWaz8XaOx52pCIqCQ2c1oKME4FHzt2LL744gt069YNy5Ytw8aNG3HhwgX4+PhgzJgx8Pf3x4IFCwAAH330EWbNmoUffvgBPXr0MH2Ok5MTnJzK/zGus+HmQTkZQMyRwp6duDPm7yvsgAZdi1xMsDOg5KmVqkjKyMGZWyk4czMZp26m4MzNFMSlZhdrZ6cQ0MLH2TSGp0MDV7TwcYbKziouR0VEJCmbCjcAsGLFCnz88ceIi4tDcHAwli9fjpCQEABA7969ERgYiLVr1wIAAgMDcePGjWKfMXv2bMyZM6fcbTHclCIj0XgfrIKwk/zA37HKCQh82Bh0GvcCvFtzvE41JKRm48ytFJy+WdjDcy8jp1g7lVKB1n7OxlNa/m7oEOCKZvWcYMfbRBBRHWNz4aY2MdxUUFK0+cUEsx64cq+Tj/nFBF0bSFGlbIiiiDsp2fljd5LzQ08KUrJyi7XV2CvQtr7xVFbBtPQmXk5QKBg2iUi+GG7KwHBTBQYDEH+myMUEjwB5D4wj8WxeGHQCewIObrVfp8yIoojYpCycvpVsmqV19lYq0nV5xdo6qpRolx922jdwQ1ADVzT04FWWiUg+GG7KwHBjAXk6IPZoYdi5fQIQi9zzSlAA9TsWjtdp0A2wL34NIqo8g0FE9L0MnMnv2Tlzyxh4snL1xdq6aOyM199p4Gq60rK/mwMDDxHZJIabMjDc1ICsZOD6X4WnsRIvmb9v51DkYoK9Ad8Oxqsvk0Xk6Q24ejcDp28m48ytFJy6mYKoO6nIySt+k1VPR1WRsGMctOzjwuBJRNaP4aYMDDe1IOVWkfE6+4F08+sYwcGjyMUEewMejWu/RpnLyTPgUnyaadDymVvJuHAnDXklXGXZx0WN9kUuOtjB3xWeTnXk2lBEZDMYbsrAcFPLRNF4W4hr+WHn+l9ATpp5G7eGhUGncS/A0UuCQuUvO1ePC3FpOHOzcMDy5YQ0lJB34O/mUCTsGG8t4arlpQCISDoMN2VguJGYPhe4daKwV+fmseIXE/RpbxyY3KQP0CgUUDlKUWmdkJmTh/O3U01XWD59MxnXEjNKvK1EoKfW7E7p7fxd4aTmVZaJqHYw3JSB4cbK6NLNLyYYf9b8fYU9EBBS2LNTvyOg5A9qTUrLzsXZW6lmU9JjkjKLtRME420lggoGLTdwRRs/VzioeJVlIrI8hpsyMNxYufS75uN1UmLN31e7FF5MsElvwKsFLyZYC5IzcwrH7+T38txKLvm2Ei18nI3X4Akw9vK08nOG2o6Bh4iqh+GmDAw3NkQUgaRrhUEn+iCQnWzextmvcKxOk16AS/3ar7OOupumw9kiA5ZP3UzB3TRdsXb2SgEtfZ2N99HKH8fTwscZ9rzKMhFVAsNNGRhubJhBD8SdNr+YoP6BH1OvloW9OoE9AI1r7ddZR4miiPhUnWlKesGtJe5nFr/KsspOgTZ+Lmb30WpazwlKXmWZiErBcFMGhhsZyc0CYv9XOBPr9kkARf5zFpSAf6ciFxPsCthxinNtEkURN+9nmU1JP30zBWnZxa+y7GCvRDt/F7M7pQd6OvK2EkQEgOGmTAw3MpaZZJxqXtCzk3TV/H17LdCoe2HY8W7LiwlKwGAQcSMp09jDczMFp2+l4OytFGTmFL/KsrParshtJVwR1MANDdx5lWWiuojhpgwMN3VIcqz54OSMu+bvaz2L3PyzN+DeqPZrJACA3iDi2t10synp526nQlfCVZbdtPamm4YW9PL4umgYeIhkjuGmDAw3dZQoAgnnC+9yfv0vIDfDvI17Y6BxT8DJ13htHZWjsben4LnptROg0ha+p+TF7WpCnt6AS/HpplNZZ24ZbyuRqy/+vywvJ3V+2HFFaz8X1HNWw8tJBQ9HFZzUdgw+RDLAcFMGhhsCAOTlALeOm19MUCx+WqRClKr84ONoDD2m5/mviz43a+dkHp4eDFJ2Gk5zf4AuT4+LcWmmKemnb6XgUnwa9CVdZjmfSqmAZ37Q8XBUwctJDQ9HFTydVPB0VMHDUV3kOcMQkbViuCkDww2VSJcG3DhsHKCcnQLkZBgfuZmFz02vM4Gc9KqHoQoTyu4xKikUmbUrGqoKglT+chldCDE7V4/zd1JxOjYZp2+l4GpCOu5l5CApI6fEcTzlUdkp4Jkffjwc1cbnjip45AcgT0c1PJxU8Mr/01GlZBgiqgUMN2VguCGLEEVAn/NA6EnPDz4ZxlNeZs8zCkNR0YBkCk+Zhe3ysmu+fqW6nIBU1mm5gucl9E7Zqa2qtykrR497GTokZeTgXnoO7mXk4F56/utiz3OQlVu1MOSVH348HNXG544qeDqpTb1Bxp4hhiGi6qjM77d8/vlGVJsEwfhDbqcGtB6W/WyDvnivUdHwVNEgZRaq8t8T8wfo6nVAlg7ISrJs7YLCvJeozFN0RduVc4rOXgsoKn+VYweVEg1UWjRw11aofWZOHu6lG3t9kjJykPhA+EnK0Jme38vQITvXgJw8A26nZON2SsVCqdrUM5R/euzBXiIn89Nn2toKQ6JofEA0/nci5v9pev3gMrECbYq+RgXaFGkrKAAIxj8FhfE7V+IyoYLtFIWfW3SZWdsH1iGbxXBDZG0USkDjYnxYkigCeboyepMqEqRKaVdwMUXRAOhSjQ9Ls9MUD0sljXWyU6Pwx7foj2r5P8RaEdCKBgSU1EYQAUcDoBUBL+Pn5unzkJurR45ej9y8POTmGZCXl4dcvQF5en3hI88AvUEPiAYIABRZBghZgHA3/zUK/hQh5D/PggG3ASgFEXYKwE4AlArAThChFEQoBeN7ioI/TesXqb3UEFLC3wWVQCg/BJnalNTuwbAllLCsiu3Mwlp5NVa0nQVrdG0AdBpTq0erKIYborpCEAB7jfEBT8t+tj6vhB6kB3uTShvDVM57BRdmzMvOP2V3z7K1V4Nd/sOhIo3zOxiqJL9DBdacQUr6wSz2AyqU38b0l1ReD5JYwrIS2hW9sGelPRCSqeIadGO4ISIbp7QDlK6Wv92FKBqvRF2Z03J5OhT713SZpy0ePLVRxr9uy/xXejV/0IvWIiggAsjOA9Ky85CcrUdqdh5SsvRIydYjJTsPyVn5j+w8pGTmISlLjxy9CEN+349Y5GEo8icgwCAWLlPZK+GmVcPNUQ13rQquWg3cnVRw12rg5qiCh5MaHo4OcHdUwdPZAQ4qu1L+/qyYWIEQZNbTVbGevoqfmjPkB9SKtKtEcCv3dF9t7Fsp7dykvW4Yww0RWS9ByD/lpAUcvaSuplYJMPYIOQDwrkB7URSRkaNHUv54oIKxQ4kZOiSZnuePG8p/nZNnAHKB2BQAKQCQm/8onYO9Mn9MUMH0+sJrCpU0iNpBZQV3hBcEQFACsIJaqFYw3BARyYAgCHBS28FJbYeGnuUPoC4IQ/fSjYOkTaHINHj6gRll6TnI0RuQlavHreQs3ErOqlBdDvZK03WEyhtEbTVhiGweww0RUR1UNAw18nQst70oikjX5eXPIiuYUaYzPTeFpCLhqCAM3byfhZv3KxaGtCol3Bzs4ai2g2N+fY5qJRzVdnDOX1a43A5O+e85FVlW8Jp3ma+7GG6IiKhcgiDAWWMPZ419hcNQmi4vv0eo+DWFkor1EumQqxeRmaOv0sUXS6KxV5QYekyhSGUHJ82D7xuXO6rt4KwpXEdtp+D1iWwIww0REVmcIAhw0djDRWOPQK+Kh6F76TlIzcpFui4P6bo8ZOQ/0nX6/D/zTH8Wvq83Ps8xvi64/1h2rgHZucaepupSKgQ4qpSF4aggFKnMe5BMoUhl3vNUdD1HFXuVahrDDRERSa5oGKouXZ4e6dnmoadoUErLf+/B5elFglLB8oJeJL1BRGp2HlKz86pdH2Aci1QsFD3Yg2TW01S0h0lptpy9SsUx3BARkayo7ZRQOynh6VT9z9IbRGTmFAlKxXqNzHuV0s2C0gMBKzsPefk3ec3K1SMrV4/E9OrXaKcQio1PKuhVKjztZr7c/LRbYcCSS68Sww0REVEplIrCsUbVJYoidHmG4qGnmr1KeQYRKVm5SMkqexp/RTnYK81Dkcp8XFLhaTfzHqSiPUwuDnZw06osUk9VMNwQERHVAkEQoLFXQmNfM71KxXuNqterdDdNV+Xa2vu74tfJD1d/J6uI4YaIiMgG1WavUnp2Ye9RSb1KRXuWMnR5cFJLGy8YboiIiOo4S/cqiWJ17ulVfQpJt05ERESyI/XsLYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVqwg3K1euRGBgIDQaDUJCQnD06NEy22/atAmtWrWCRqNB+/btsWPHjlqqlIiIiKyd5OFmw4YNCA8Px+zZs3HixAkEBQWhX79+SEhIKLH94cOHMWrUKLz44os4efIkhgwZgiFDhuDs2bO1XDkRERFZI0EURVHKAkJCQtC1a1esWLECAGAwGBAQEIDJkydj+vTpxdqPGDECGRkZ+O2330zLHnroIQQHB2P16tXlbi81NRWurq5ISUmBi4uL5XaEiIiIakxlfr8l7bnJycnB8ePHERYWZlqmUCgQFhaGI0eOlLjOkSNHzNoDQL9+/UptT0RERHWLnZQbT0xMhF6vh4+Pj9lyHx8fXLhwocR14uLiSmwfFxdXYnudTgedTmd6nZKSAsCYAImIiMg2FPxuV+SEk6ThpjYsWLAAc+fOLbY8ICBAgmqIiIioOtLS0uDq6lpmG0nDjZeXF5RKJeLj482Wx8fHw9fXt8R1fH19K9V+xowZCA8PN702GAxISkqCp6cnBEGo5h6YS01NRUBAAGJjY2U5nkfu+wfIfx+5f7ZP7vvI/bN9NbWPoigiLS0N9evXL7etpOFGpVKhc+fO2Lt3L4YMGQLAGD727t2LSZMmlbhOaGgo9u7di6lTp5qWRUREIDQ0tMT2arUaarXabJmbm5slyi+Vi4uLbP+jBeS/f4D895H7Z/vkvo/cP9tXE/tYXo9NAclPS4WHh2Ps2LHo0qULunXrhmXLliEjIwPjx48HAIwZMwb+/v5YsGABAGDKlCno1asXlixZgoEDB2L9+vX4559/sGbNGil3g4iIiKyE5OFmxIgRuHv3LmbNmoW4uDgEBwdj165dpkHDMTExUCgKJ3V1794dP/zwA9577z288847aN68ObZu3Yp27dpJtQtERERkRSQPNwAwadKkUk9D7d+/v9iyZ555Bs8880wNV1V5arUas2fPLnYaTC7kvn+A/PeR+2f75L6P3D/bZw37KPlF/IiIiIgsSfLbLxARERFZEsMNERERyQrDDREREckKww0RERHJCsNNJa1cuRKBgYHQaDQICQnB0aNHy2y/adMmtGrVChqNBu3bt8eOHTtqqdKqqcz+rV27FoIgmD00Gk0tVls5Bw8exKBBg1C/fn0IgoCtW7eWu87+/fvRqVMnqNVqNGvWDGvXrq3xOquqsvu3f//+YsdPEIRS79MmtQULFqBr165wdnaGt7c3hgwZgosXL5a7ni19B6uyj7b0PVy1ahU6dOhgurhbaGgodu7cWeY6tnT8Krt/tnTsSrJw4UIIgmB2Ud2SSHEMGW4qYcOGDQgPD8fs2bNx4sQJBAUFoV+/fkhISCix/eHDhzFq1Ci8+OKLOHnyJIYMGYIhQ4bg7NmztVx5xVR2/wDjFSjv3Lljety4caMWK66cjIwMBAUFYeXKlRVqHx0djYEDB6JPnz6IjIzE1KlT8dJLL2H37t01XGnVVHb/Cly8eNHsGHp7e9dQhdVz4MABTJw4EX///TciIiKQm5uLxx9/HBkZGaWuY2vfwarsI2A738MGDRpg4cKFOH78OP755x88+uijGDx4MM6dO1die1s7fpXdP8B2jt2Djh07hi+++AIdOnQos51kx1CkCuvWrZs4ceJE02u9Xi/Wr19fXLBgQYnthw8fLg4cONBsWUhIiPjvf/+7Ruusqsru3zfffCO6urrWUnWWBUDcsmVLmW3eeustsW3btmbLRowYIfbr168GK7OMiuzfvn37RADi/fv3a6UmS0tISBABiAcOHCi1ja19Bx9UkX205e+hKIqiu7u7+J///KfE92z9+Ili2ftnq8cuLS1NbN68uRgRESH26tVLnDJlSqltpTqG7LmpoJycHBw/fhxhYWGmZQqFAmFhYThy5EiJ6xw5csSsPQD069ev1PZSqsr+AUB6ejoaNWqEgICAcv+FYmts6fhVR3BwMPz8/PDYY4/h0KFDUpdTYSkpKQAADw+PUtvY+jGsyD4Ctvk91Ov1WL9+PTIyMkq9N6AtH7+K7B9gm8du4sSJGDhwYLFjUxKpjiHDTQUlJiZCr9ebbgtRwMfHp9QxCnFxcZVqL6Wq7F/Lli3x9ddf45dffsF///tfGAwGdO/eHTdv3qyNkmtcaccvNTUVWVlZElVlOX5+fli9ejV++ukn/PTTTwgICEDv3r1x4sQJqUsrl8FgwNSpU9GjR48yb71iS9/BB1V0H23te3jmzBk4OTlBrVbjlVdewZYtW9CmTZsS29ri8avM/tnasQOA9evX48SJE6b7PZZHqmNoFbdfINsUGhpq9i+S7t27o3Xr1vjiiy8wf/58CSujimjZsiVatmxpet29e3dcvXoVn3zyCb7//nsJKyvfxIkTcfbsWfz1119Sl1JjKrqPtvY9bNmyJSIjI5GSkoLNmzdj7NixOHDgQKkBwNZUZv9s7djFxsZiypQpiIiIsPqBzww3FeTl5QWlUon4+Hiz5fHx8fD19S1xHV9f30q1l1JV9u9B9vb26NixI65cuVITJda60o6fi4sLHBwcJKqqZnXr1s3qA8OkSZPw22+/4eDBg2jQoEGZbW3pO1hUZfbxQdb+PVSpVGjWrBkAoHPnzjh27Bg+/fRTfPHFF8Xa2uLxq8z+Pcjaj93x48eRkJCATp06mZbp9XocPHgQK1asgE6ng1KpNFtHqmPI01IVpFKp0LlzZ+zdu9e0zGAwYO/evaWeTw0NDTVrDwARERFlnn+VSlX270F6vR5nzpyBn59fTZVZq2zp+FlKZGSk1R4/URQxadIkbNmyBX/88QcaN25c7jq2dgyrso8PsrXvocFggE6nK/E9Wzt+JSlr/x5k7ceub9++OHPmDCIjI02PLl26YPTo0YiMjCwWbAAJj2GNDleWmfXr14tqtVpcu3ateP78efHll18W3dzcxLi4OFEURfH5558Xp0+fbmp/6NAh0c7OTly8eLEYFRUlzp49W7S3txfPnDkj1S6UqbL7N3fuXHH37t3i1atXxePHj4sjR44UNRqNeO7cOal2oUxpaWniyZMnxZMnT4oAxKVLl4onT54Ub9y4IYqiKE6fPl18/vnnTe2vXbsmarVa8c033xSjoqLElStXikqlUty1a5dUu1Cmyu7fJ598Im7dulW8fPmyeObMGXHKlCmiQqEQ9+zZI9UulGnChAmiq6uruH//fvHOnTumR2ZmpqmNrX8Hq7KPtvQ9nD59unjgwAExOjpaPH36tDh9+nRREATx999/F0XR9o9fZffPlo5daR6cLWUtx5DhppI+++wzsWHDhqJKpRK7desm/v3336b3evXqJY4dO9as/caNG8UWLVqIKpVKbNu2rbh9+/ZarrhyKrN/U6dONbX18fERn3jiCfHEiRMSVF0xBVOfH3wU7NPYsWPFXr16FVsnODhYVKlUYpMmTcRvvvmm1uuuqMru30cffSQ2bdpU1Gg0ooeHh9i7d2/xjz/+kKb4Cihp3wCYHRNb/w5WZR9t6Xv4wgsviI0aNRJVKpVYr149sW/fvqYfflG0/eNX2f2zpWNXmgfDjbUcQ0EURbFm+4aIiIiIag/H3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQUZ0nCAK2bt0qdRlEZCEMN0QkqXHjxkEQhGKP/v37S10aEdko3hWciCTXv39/fPPNN2bL1Gq1RNUQka1jzw0RSU6tVsPX19fs4e7uDsB4ymjVqlUYMGAAHBwc0KRJE2zevNls/TNnzuDRRx+Fg4MDPD098fLLLyM9Pd2szddff422bdtCrVbDz88PkyZNMns/MTERQ4cOhVarRfPmzbFt27aa3WkiqjEMN0Rk9WbOnIlhw4bh1KlTGD16NEaOHImoqCgAQEZGBvr16wd3d3ccO3YMmzZtwp49e8zCy6pVqzBx4kS8/PLLOHPmDLZt24ZmzZqZbWPu3LkYPnw4Tp8+jSeeeAKjR49GUlJSre4nEVlIjd+ak4ioDGPHjhWVSqXo6Oho9vjggw9EUTTeKfuVV14xWyckJEScMGGCKIqiuGbNGtHd3V1MT083vb99+3ZRoVCIcXFxoiiKYv369cV333231BoAiO+9957pdXp6ughA3Llzp8X2k4hqD8fcEJHk+vTpg1WrVpkt8/DwMD0PDQ01ey80NBSRkZEAgKioKAQFBcHR0dH0fo8ePWAwGHDx4kUIgoDbt2+jb9++ZdbQoUMH03NHR0e4uLggISGhqrtERBJiuCEiyTk6OhY7TWQpDg4OFWpnb29v9loQBBgMhpooiYhqGMfcEJHV+/vvv4u9bt26NQCgdevWOHXqFDIyMkzvHzp0CAqFAi1btoSzszMCAwOxd+/eWq2ZiKTDnhsikpxOp0NcXJzZMjs7O3h5eQEANm3ahC5duuDhhx/GunXrcPToUXz11VcAgNGjR2P27NkYO3Ys5syZg7t372Ly5Ml4/vnn4ePjAwCYM2cOXnnlFXh7e2PAgAFIS0vDoUOHMHny5NrdUSKqFQw3RCS5Xbt2wc/Pz2xZy5YtceHCBQDGmUzr16/Hq6++Cj8/P/z4449o06YNAECr1WL37t2YMmUKunbtCq1Wi2HDhmHp0qWmzxo7diyys7PxySefYNq0afDy8sLTTz9deztIRLVKEEVRlLoIIqLSCIKALVu2YMiQIVKXQkQ2gmNuiIiISFYYboiIiEhWOOaGiKwaz5wTUWWx54aIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGTl/wFuK6UwtY28qAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = corpus.test\n",
        "sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EYc8ntl4-SB8",
        "outputId": "e9f7c9d5-98b2-454e-fac7-c9ae5c89866a"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch.utils.data.dataset.ConcatDataset object at 0x7891749e7220>"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# load tagger\n",
        "tagger: SequenceTagger = SequenceTagger.load('models/taggers/ner/202309241957/final-model.pt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "psStECkDwXyv",
        "outputId": "2581ddc7-ae2b-4754-d568-9fd98b60ef94"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:10:48,169 SequenceTagger predicts: Dictionary with 9 tags: O, B-PERSON, I-PERSON, B-ORG, I-ORG, B-LOCATION, I-LOCATION, B-DATE, I-DATE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_test_labels(sentences):\n",
        "    \"\"\"\n",
        "    Get test labels\n",
        "    \"\"\"\n",
        "    test_labels = []\n",
        "    for sentence in sentences:\n",
        "        sentence_labels = [\"O\"] * len(sentence)\n",
        "        for label in sentence.get_labels('ner'):\n",
        "            span = label.data_point\n",
        "            if len(span) == 1:\n",
        "                sentence_labels[span[0].idx - 1] = \"B-\" + label.value\n",
        "            else:\n",
        "                sentence_labels[span[0].idx - 1] = \"B-\" + label.value\n",
        "                sentence_labels[span[-1].idx - 1] = \"I-\" + label.value\n",
        "                for i in range(span[0].idx, span[-1].idx - 1):\n",
        "                    sentence_labels[i] = \"I-\" + label.value\n",
        "\n",
        "        test_labels.append(sentence_labels)\n",
        "    return test_labels\n",
        "\n",
        "def get_pred_labels(sentences, tagger):\n",
        "    \"\"\"\n",
        "    Get predicted labels\n",
        "    \"\"\"\n",
        "    tagger.predict(sentences, mini_batch_size=32, verbose=True,  embedding_storage_mode=\"none\")\n",
        "\n",
        "    pred_labels = []\n",
        "    for sentence in sentences:\n",
        "        sentence_labels = [\"O\"] * len(sentence)\n",
        "        for label in sentence.get_labels('ner'):\n",
        "            span = label.data_point\n",
        "            if len(span) == 1:\n",
        "                sentence_labels[span[0].idx - 1] = \"B-\" + label.value\n",
        "            else:\n",
        "                sentence_labels[span[0].idx - 1] = \"B-\" + label.value\n",
        "                sentence_labels[span[-1].idx - 1] = \"I-\" + label.value\n",
        "                for i in range(span[0].idx, span[-1].idx - 1):\n",
        "                    sentence_labels[i] = \"I-\" + label.value\n",
        "\n",
        "        pred_labels.append(sentence_labels)\n",
        "    return pred_labels"
      ],
      "metadata": {
        "id": "IHp0g88BHpjL"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%pprint\n",
        "test_labels = get_test_labels(sentences)\n",
        "test_labels[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1nhP6V3rIQt-",
        "outputId": "87c1fd7d-ee2d-44e0-fe01-a6bf522ba91d"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pretty printing has been turned OFF\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'], ['B-PERSON', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'B-PERSON', 'B-ORG', 'I-ORG', 'O', 'O'], ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'B-PERSON', 'O', 'B-ORG', 'I-ORG', 'O', 'B-ORG', 'I-ORG', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOCATION', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'B-PERSON', 'B-PERSON', 'B-PERSON', 'B-PERSON']]"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred_labels = get_pred_labels(sentences, tagger)\n",
        "pred_labels[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j07LtnezIXbw",
        "outputId": "85124096-f7c7-4b2d-a8e1-c9a3dd13335a"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Batch inference: 100%|██████████| 3/3 [00:02<00:00,  1.16it/s]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DATE', 'I-DATE', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DATE', 'O', 'B-PERSON', 'B-ORG', 'O', 'O', 'B-PERSON', 'O', 'O', 'B-ORG', 'B-LOCATION', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'], ['B-PERSON', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O'], ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'B-ORG', 'B-LOCATION', 'O', 'B-ORG', 'I-ORG', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOCATION', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'B-DATE', 'I-DATE', 'B-DATE', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'B-PERSON', 'I-PERSON', 'B-PERSON', 'I-PERSON']]"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(test_labels), len(pred_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E8CxfQfgIXZw",
        "outputId": "c198fcc3-d274-494a-fd6b-7b35abd0b71a"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(96, 96)"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "\n",
        "_test_labels = numpy.concatenate([numpy.array(i) for i in test_labels])\n",
        "_test_labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fACMbO7pE-iC",
        "outputId": "34a23b39-eba5-4e59-edae-354c5f9e898d"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['O', 'O', 'O', ..., 'O', 'O', 'B-PERSON'], dtype='<U10')"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "_pred_labels = numpy.concatenate([numpy.array(i) for i in pred_labels])\n",
        "_pred_labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pyVXk3cU-NG1",
        "outputId": "e2014a29-fcf2-4949-b360-54d6f5f099fe"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['B-PERSON', 'O', 'O', ..., 'O', 'O', 'B-PERSON'], dtype='<U10')"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import classification_report\n",
        "\n",
        "labels = ['B-DATE', 'B-LOCATION', 'B-ORG', 'B-PERSON', 'I-DATE', 'I-LOCATION', 'I-ORG', 'I-PERSON']\n",
        "print(classification_report(_test_labels, _pred_labels, labels=labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AHLtR8Ob-qvM",
        "outputId": "5cf99957-bf1e-4b8f-a5de-54180befd34d"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "      B-DATE       0.33      0.40      0.36        53\n",
            "  B-LOCATION       0.57      0.57      0.57        74\n",
            "       B-ORG       0.55      0.67      0.60       149\n",
            "    B-PERSON       0.78      0.84      0.81       201\n",
            "      I-DATE       0.57      0.28      0.38        46\n",
            "  I-LOCATION       0.50      0.30      0.37        30\n",
            "       I-ORG       0.54      0.61      0.58        70\n",
            "    I-PERSON       0.80      0.84      0.82        68\n",
            "\n",
            "   micro avg       0.62      0.66      0.64       691\n",
            "   macro avg       0.58      0.56      0.56       691\n",
            "weighted avg       0.62      0.66      0.63       691\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The current date will serve as the model version number\n",
        "\n",
        "model_version = get_current_date(date_format=\"%Y%m%d%H%M\")\n",
        "model_version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "7mF2JITq0zC3",
        "outputId": "2544b012-72f9-4871-abec-371825a742ba"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'202309242015'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "LR = 5e-5\n",
        "BATCH_SIZE = 16\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "trainer.fine_tune(f\"models/taggers/ner/{model_version}\",\n",
        "              learning_rate=LR,\n",
        "              mini_batch_size=BATCH_SIZE,\n",
        "              max_epochs=NUM_EPOCHS,\n",
        "              optimizer=torch.optim.AdamW,\n",
        "              main_evaluation_metric=(\"micro avg\", \"f1-score\")\n",
        "              )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e4vclCcu-A2_",
        "outputId": "1e277952-8f35-4e3c-e87e-60c1bcf3b8e0"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:16:35,103 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:35,108 Model: \"SequenceTagger(\n",
            "  (embeddings): TransformerWordEmbeddings(\n",
            "    (model): BertModel(\n",
            "      (embeddings): BertEmbeddings(\n",
            "        (word_embeddings): Embedding(30523, 768)\n",
            "        (position_embeddings): Embedding(512, 768)\n",
            "        (token_type_embeddings): Embedding(2, 768)\n",
            "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (encoder): BertEncoder(\n",
            "        (layer): ModuleList(\n",
            "          (0-11): 12 x BertLayer(\n",
            "            (attention): BertAttention(\n",
            "              (self): BertSelfAttention(\n",
            "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (dropout): Dropout(p=0.1, inplace=False)\n",
            "              )\n",
            "              (output): BertSelfOutput(\n",
            "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "                (dropout): Dropout(p=0.1, inplace=False)\n",
            "              )\n",
            "            )\n",
            "            (intermediate): BertIntermediate(\n",
            "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (intermediate_act_fn): GELUActivation()\n",
            "            )\n",
            "            (output): BertOutput(\n",
            "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (pooler): BertPooler(\n",
            "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "        (activation): Tanh()\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (locked_dropout): LockedDropout(p=0.5)\n",
            "  (linear): Linear(in_features=768, out_features=9, bias=True)\n",
            "  (loss_function): CrossEntropyLoss()\n",
            ")\"\n",
            "2023-09-24 20:16:35,111 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:35,114 Corpus: \"Corpus: 768 train + 96 dev + 96 test sentences\"\n",
            "2023-09-24 20:16:35,118 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:35,122 Parameters:\n",
            "2023-09-24 20:16:35,124  - learning_rate: \"0.000050\"\n",
            "2023-09-24 20:16:35,125  - mini_batch_size: \"16\"\n",
            "2023-09-24 20:16:35,128  - patience: \"3\"\n",
            "2023-09-24 20:16:35,129  - anneal_factor: \"0.5\"\n",
            "2023-09-24 20:16:35,130  - max_epochs: \"10\"\n",
            "2023-09-24 20:16:35,132  - shuffle: \"True\"\n",
            "2023-09-24 20:16:35,134  - train_with_dev: \"False\"\n",
            "2023-09-24 20:16:35,135  - batch_growth_annealing: \"False\"\n",
            "2023-09-24 20:16:35,137 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:35,140 Model training base path: \"models/taggers/ner/202309242015\"\n",
            "2023-09-24 20:16:35,141 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:35,143 Device: cuda:0\n",
            "2023-09-24 20:16:35,145 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:35,147 Embeddings storage mode: none\n",
            "2023-09-24 20:16:35,148 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:16:39,566 epoch 1 - iter 4/48 - loss 2.47994771 - time (sec): 4.42 - samples/sec: 1363.75 - lr: 0.000004\n",
            "2023-09-24 20:16:42,867 epoch 1 - iter 8/48 - loss 2.35055433 - time (sec): 7.72 - samples/sec: 1576.58 - lr: 0.000008\n",
            "2023-09-24 20:16:46,840 epoch 1 - iter 12/48 - loss 2.16049971 - time (sec): 11.69 - samples/sec: 1541.28 - lr: 0.000013\n",
            "2023-09-24 20:16:51,942 epoch 1 - iter 16/48 - loss 1.86888440 - time (sec): 16.79 - samples/sec: 1448.66 - lr: 0.000017\n",
            "2023-09-24 20:16:56,504 epoch 1 - iter 20/48 - loss 1.58651471 - time (sec): 21.35 - samples/sec: 1434.32 - lr: 0.000021\n",
            "2023-09-24 20:17:00,470 epoch 1 - iter 24/48 - loss 1.40381384 - time (sec): 25.32 - samples/sec: 1448.48 - lr: 0.000025\n",
            "2023-09-24 20:17:04,258 epoch 1 - iter 28/48 - loss 1.26159681 - time (sec): 29.11 - samples/sec: 1468.52 - lr: 0.000029\n",
            "2023-09-24 20:17:08,306 epoch 1 - iter 32/48 - loss 1.15589624 - time (sec): 33.15 - samples/sec: 1470.92 - lr: 0.000033\n",
            "2023-09-24 20:17:12,455 epoch 1 - iter 36/48 - loss 1.08007918 - time (sec): 37.30 - samples/sec: 1471.71 - lr: 0.000038\n",
            "2023-09-24 20:17:17,081 epoch 1 - iter 40/48 - loss 1.00387507 - time (sec): 41.93 - samples/sec: 1458.48 - lr: 0.000042\n",
            "2023-09-24 20:17:20,772 epoch 1 - iter 44/48 - loss 0.94590436 - time (sec): 45.62 - samples/sec: 1472.13 - lr: 0.000046\n",
            "2023-09-24 20:17:24,949 epoch 1 - iter 48/48 - loss 0.89167027 - time (sec): 49.80 - samples/sec: 1471.39 - lr: 0.000050\n",
            "2023-09-24 20:17:24,954 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:17:24,955 EPOCH 1 done: loss 0.8917 - lr 0.000050\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.51it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:17:28,943 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:17:28,976 DEV : loss 0.250569224357605 - f1-score (micro avg)  0.2935\n",
            "2023-09-24 20:17:28,990 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:17:32,857 epoch 2 - iter 4/48 - loss 0.27881660 - time (sec): 3.86 - samples/sec: 1598.40 - lr: 0.000050\n",
            "2023-09-24 20:17:37,071 epoch 2 - iter 8/48 - loss 0.26397586 - time (sec): 8.08 - samples/sec: 1510.91 - lr: 0.000049\n",
            "2023-09-24 20:17:41,446 epoch 2 - iter 12/48 - loss 0.26867236 - time (sec): 12.45 - samples/sec: 1457.11 - lr: 0.000049\n",
            "2023-09-24 20:17:45,218 epoch 2 - iter 16/48 - loss 0.25900080 - time (sec): 16.22 - samples/sec: 1488.02 - lr: 0.000048\n",
            "2023-09-24 20:17:49,570 epoch 2 - iter 20/48 - loss 0.24929739 - time (sec): 20.58 - samples/sec: 1477.13 - lr: 0.000048\n",
            "2023-09-24 20:17:54,814 epoch 2 - iter 24/48 - loss 0.24138383 - time (sec): 25.82 - samples/sec: 1418.03 - lr: 0.000047\n",
            "2023-09-24 20:17:59,731 epoch 2 - iter 28/48 - loss 0.23666154 - time (sec): 30.74 - samples/sec: 1384.70 - lr: 0.000047\n",
            "2023-09-24 20:18:04,096 epoch 2 - iter 32/48 - loss 0.23405549 - time (sec): 35.10 - samples/sec: 1381.46 - lr: 0.000046\n",
            "2023-09-24 20:18:09,022 epoch 2 - iter 36/48 - loss 0.22837248 - time (sec): 40.03 - samples/sec: 1365.50 - lr: 0.000046\n",
            "2023-09-24 20:18:13,657 epoch 2 - iter 40/48 - loss 0.22585951 - time (sec): 44.66 - samples/sec: 1357.56 - lr: 0.000045\n",
            "2023-09-24 20:18:19,186 epoch 2 - iter 44/48 - loss 0.21939116 - time (sec): 50.19 - samples/sec: 1334.57 - lr: 0.000045\n",
            "2023-09-24 20:18:24,106 epoch 2 - iter 48/48 - loss 0.21932473 - time (sec): 55.11 - samples/sec: 1329.49 - lr: 0.000045\n",
            "2023-09-24 20:18:24,116 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:18:24,119 EPOCH 2 done: loss 0.2193 - lr 0.000045\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:06<00:00,  1.02s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:18:30,279 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:18:30,315 DEV : loss 0.15357275307178497 - f1-score (micro avg)  0.5639\n",
            "2023-09-24 20:18:30,333 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:18:34,205 epoch 3 - iter 4/48 - loss 0.18280439 - time (sec): 3.87 - samples/sec: 1595.68 - lr: 0.000044\n",
            "2023-09-24 20:18:38,066 epoch 3 - iter 8/48 - loss 0.17321362 - time (sec): 7.73 - samples/sec: 1577.19 - lr: 0.000044\n",
            "2023-09-24 20:18:42,448 epoch 3 - iter 12/48 - loss 0.16478259 - time (sec): 12.11 - samples/sec: 1508.75 - lr: 0.000043\n",
            "2023-09-24 20:18:46,998 epoch 3 - iter 16/48 - loss 0.15777427 - time (sec): 16.66 - samples/sec: 1464.70 - lr: 0.000043\n",
            "2023-09-24 20:18:51,407 epoch 3 - iter 20/48 - loss 0.15594970 - time (sec): 21.07 - samples/sec: 1436.57 - lr: 0.000042\n",
            "2023-09-24 20:18:55,700 epoch 3 - iter 24/48 - loss 0.15550636 - time (sec): 25.36 - samples/sec: 1436.75 - lr: 0.000042\n",
            "2023-09-24 20:19:01,429 epoch 3 - iter 28/48 - loss 0.15392499 - time (sec): 31.09 - samples/sec: 1367.83 - lr: 0.000041\n",
            "2023-09-24 20:19:06,341 epoch 3 - iter 32/48 - loss 0.15398648 - time (sec): 36.01 - samples/sec: 1348.64 - lr: 0.000041\n",
            "2023-09-24 20:19:10,639 epoch 3 - iter 36/48 - loss 0.15327585 - time (sec): 40.30 - samples/sec: 1354.22 - lr: 0.000040\n",
            "2023-09-24 20:19:14,909 epoch 3 - iter 40/48 - loss 0.15600147 - time (sec): 44.57 - samples/sec: 1364.51 - lr: 0.000040\n",
            "2023-09-24 20:19:19,871 epoch 3 - iter 44/48 - loss 0.15332635 - time (sec): 49.54 - samples/sec: 1358.15 - lr: 0.000040\n",
            "2023-09-24 20:19:23,973 epoch 3 - iter 48/48 - loss 0.15230361 - time (sec): 53.64 - samples/sec: 1366.05 - lr: 0.000039\n",
            "2023-09-24 20:19:23,983 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:19:23,994 EPOCH 3 done: loss 0.1523 - lr 0.000039\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:04<00:00,  1.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:19:28,197 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:19:28,224 DEV : loss 0.13580676913261414 - f1-score (micro avg)  0.5925\n",
            "2023-09-24 20:19:28,238 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:19:33,192 epoch 4 - iter 4/48 - loss 0.11812275 - time (sec): 4.95 - samples/sec: 1232.67 - lr: 0.000039\n",
            "2023-09-24 20:19:37,753 epoch 4 - iter 8/48 - loss 0.12271959 - time (sec): 9.51 - samples/sec: 1288.00 - lr: 0.000038\n",
            "2023-09-24 20:19:41,338 epoch 4 - iter 12/48 - loss 0.12172099 - time (sec): 13.09 - samples/sec: 1410.40 - lr: 0.000038\n",
            "2023-09-24 20:19:45,372 epoch 4 - iter 16/48 - loss 0.11917413 - time (sec): 17.13 - samples/sec: 1436.74 - lr: 0.000037\n",
            "2023-09-24 20:19:50,289 epoch 4 - iter 20/48 - loss 0.12001025 - time (sec): 22.05 - samples/sec: 1397.51 - lr: 0.000037\n",
            "2023-09-24 20:19:54,738 epoch 4 - iter 24/48 - loss 0.11968905 - time (sec): 26.49 - samples/sec: 1393.58 - lr: 0.000036\n",
            "2023-09-24 20:19:59,469 epoch 4 - iter 28/48 - loss 0.11519499 - time (sec): 31.23 - samples/sec: 1370.47 - lr: 0.000036\n",
            "2023-09-24 20:20:03,552 epoch 4 - iter 32/48 - loss 0.11330544 - time (sec): 35.31 - samples/sec: 1383.74 - lr: 0.000035\n",
            "2023-09-24 20:20:07,666 epoch 4 - iter 36/48 - loss 0.11458711 - time (sec): 39.42 - samples/sec: 1396.50 - lr: 0.000035\n",
            "2023-09-24 20:20:12,202 epoch 4 - iter 40/48 - loss 0.11892615 - time (sec): 43.96 - samples/sec: 1390.44 - lr: 0.000035\n",
            "2023-09-24 20:20:16,453 epoch 4 - iter 44/48 - loss 0.11773453 - time (sec): 48.21 - samples/sec: 1395.92 - lr: 0.000034\n",
            "2023-09-24 20:20:20,881 epoch 4 - iter 48/48 - loss 0.11594640 - time (sec): 52.64 - samples/sec: 1392.01 - lr: 0.000034\n",
            "2023-09-24 20:20:20,887 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:20:20,890 EPOCH 4 done: loss 0.1159 - lr 0.000034\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:20:24,691 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:20:24,714 DEV : loss 0.13611148297786713 - f1-score (micro avg)  0.5987\n",
            "2023-09-24 20:20:24,729 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:20:28,726 epoch 5 - iter 4/48 - loss 0.08270656 - time (sec): 3.99 - samples/sec: 1518.65 - lr: 0.000033\n",
            "2023-09-24 20:20:32,386 epoch 5 - iter 8/48 - loss 0.08480999 - time (sec): 7.65 - samples/sec: 1595.20 - lr: 0.000033\n",
            "2023-09-24 20:20:36,306 epoch 5 - iter 12/48 - loss 0.08386787 - time (sec): 11.57 - samples/sec: 1596.88 - lr: 0.000032\n",
            "2023-09-24 20:20:40,704 epoch 5 - iter 16/48 - loss 0.08961045 - time (sec): 15.97 - samples/sec: 1540.24 - lr: 0.000032\n",
            "2023-09-24 20:20:45,851 epoch 5 - iter 20/48 - loss 0.08965993 - time (sec): 21.12 - samples/sec: 1458.89 - lr: 0.000031\n",
            "2023-09-24 20:20:50,449 epoch 5 - iter 24/48 - loss 0.08840117 - time (sec): 25.72 - samples/sec: 1443.27 - lr: 0.000031\n",
            "2023-09-24 20:20:54,692 epoch 5 - iter 28/48 - loss 0.08709687 - time (sec): 29.96 - samples/sec: 1436.77 - lr: 0.000030\n",
            "2023-09-24 20:20:59,967 epoch 5 - iter 32/48 - loss 0.08672790 - time (sec): 35.23 - samples/sec: 1393.03 - lr: 0.000030\n",
            "2023-09-24 20:21:03,603 epoch 5 - iter 36/48 - loss 0.08793902 - time (sec): 38.87 - samples/sec: 1417.30 - lr: 0.000030\n",
            "2023-09-24 20:21:07,482 epoch 5 - iter 40/48 - loss 0.08780340 - time (sec): 42.75 - samples/sec: 1430.78 - lr: 0.000029\n",
            "2023-09-24 20:21:12,039 epoch 5 - iter 44/48 - loss 0.08697574 - time (sec): 47.31 - samples/sec: 1421.13 - lr: 0.000029\n",
            "2023-09-24 20:21:16,266 epoch 5 - iter 48/48 - loss 0.08728031 - time (sec): 51.53 - samples/sec: 1421.86 - lr: 0.000028\n",
            "2023-09-24 20:21:16,273 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:21:16,275 EPOCH 5 done: loss 0.0873 - lr 0.000028\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.54it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:21:20,198 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:21:20,222 DEV : loss 0.13003411889076233 - f1-score (micro avg)  0.6258\n",
            "2023-09-24 20:21:20,235 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:21:24,152 epoch 6 - iter 4/48 - loss 0.04858793 - time (sec): 3.92 - samples/sec: 1542.19 - lr: 0.000028\n",
            "2023-09-24 20:21:28,239 epoch 6 - iter 8/48 - loss 0.06353514 - time (sec): 8.00 - samples/sec: 1536.31 - lr: 0.000027\n",
            "2023-09-24 20:21:31,927 epoch 6 - iter 12/48 - loss 0.06635338 - time (sec): 11.69 - samples/sec: 1558.54 - lr: 0.000027\n",
            "2023-09-24 20:21:36,300 epoch 6 - iter 16/48 - loss 0.06708908 - time (sec): 16.06 - samples/sec: 1513.63 - lr: 0.000026\n",
            "2023-09-24 20:21:40,314 epoch 6 - iter 20/48 - loss 0.06855053 - time (sec): 20.08 - samples/sec: 1518.07 - lr: 0.000026\n",
            "2023-09-24 20:21:44,503 epoch 6 - iter 24/48 - loss 0.06803013 - time (sec): 24.27 - samples/sec: 1506.13 - lr: 0.000026\n",
            "2023-09-24 20:21:49,539 epoch 6 - iter 28/48 - loss 0.06885310 - time (sec): 29.30 - samples/sec: 1458.41 - lr: 0.000025\n",
            "2023-09-24 20:21:54,249 epoch 6 - iter 32/48 - loss 0.06999568 - time (sec): 34.01 - samples/sec: 1433.71 - lr: 0.000025\n",
            "2023-09-24 20:21:58,851 epoch 6 - iter 36/48 - loss 0.07067953 - time (sec): 38.61 - samples/sec: 1421.33 - lr: 0.000024\n",
            "2023-09-24 20:22:02,916 epoch 6 - iter 40/48 - loss 0.07198831 - time (sec): 42.68 - samples/sec: 1429.27 - lr: 0.000024\n",
            "2023-09-24 20:22:06,889 epoch 6 - iter 44/48 - loss 0.07189223 - time (sec): 46.65 - samples/sec: 1438.84 - lr: 0.000023\n",
            "2023-09-24 20:22:10,697 epoch 6 - iter 48/48 - loss 0.07104766 - time (sec): 50.46 - samples/sec: 1452.08 - lr: 0.000023\n",
            "2023-09-24 20:22:10,704 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:22:10,709 EPOCH 6 done: loss 0.0710 - lr 0.000023\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.66it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:22:14,351 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:22:14,367 DEV : loss 0.1369745284318924 - f1-score (micro avg)  0.6306\n",
            "2023-09-24 20:22:14,376 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:22:18,608 epoch 7 - iter 4/48 - loss 0.05347093 - time (sec): 4.23 - samples/sec: 1387.24 - lr: 0.000022\n",
            "2023-09-24 20:22:22,713 epoch 7 - iter 8/48 - loss 0.05825195 - time (sec): 8.33 - samples/sec: 1418.62 - lr: 0.000022\n",
            "2023-09-24 20:22:26,992 epoch 7 - iter 12/48 - loss 0.05394096 - time (sec): 12.61 - samples/sec: 1427.61 - lr: 0.000021\n",
            "2023-09-24 20:22:31,953 epoch 7 - iter 16/48 - loss 0.05703863 - time (sec): 17.57 - samples/sec: 1371.94 - lr: 0.000021\n",
            "2023-09-24 20:22:35,831 epoch 7 - iter 20/48 - loss 0.05643927 - time (sec): 21.45 - samples/sec: 1407.79 - lr: 0.000021\n",
            "2023-09-24 20:22:40,387 epoch 7 - iter 24/48 - loss 0.05576627 - time (sec): 26.01 - samples/sec: 1397.17 - lr: 0.000020\n",
            "2023-09-24 20:22:44,433 epoch 7 - iter 28/48 - loss 0.05973428 - time (sec): 30.05 - samples/sec: 1413.06 - lr: 0.000020\n",
            "2023-09-24 20:22:49,869 epoch 7 - iter 32/48 - loss 0.05849898 - time (sec): 35.49 - samples/sec: 1368.43 - lr: 0.000019\n",
            "2023-09-24 20:22:53,960 epoch 7 - iter 36/48 - loss 0.05786501 - time (sec): 39.58 - samples/sec: 1382.35 - lr: 0.000019\n",
            "2023-09-24 20:22:57,740 epoch 7 - iter 40/48 - loss 0.05636693 - time (sec): 43.36 - samples/sec: 1402.23 - lr: 0.000018\n",
            "2023-09-24 20:23:02,527 epoch 7 - iter 44/48 - loss 0.05787139 - time (sec): 48.15 - samples/sec: 1392.71 - lr: 0.000018\n",
            "2023-09-24 20:23:07,144 epoch 7 - iter 48/48 - loss 0.05734594 - time (sec): 52.77 - samples/sec: 1388.62 - lr: 0.000017\n",
            "2023-09-24 20:23:07,148 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:23:07,150 EPOCH 7 done: loss 0.0573 - lr 0.000017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:23:10,217 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:23:10,234 DEV : loss 0.1462002843618393 - f1-score (micro avg)  0.6291\n",
            "2023-09-24 20:23:10,243 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:23:14,188 epoch 8 - iter 4/48 - loss 0.04757033 - time (sec): 3.94 - samples/sec: 1508.07 - lr: 0.000017\n",
            "2023-09-24 20:23:18,225 epoch 8 - iter 8/48 - loss 0.04481067 - time (sec): 7.98 - samples/sec: 1494.62 - lr: 0.000016\n",
            "2023-09-24 20:23:22,333 epoch 8 - iter 12/48 - loss 0.04508964 - time (sec): 12.09 - samples/sec: 1497.54 - lr: 0.000016\n",
            "2023-09-24 20:23:25,877 epoch 8 - iter 16/48 - loss 0.04629205 - time (sec): 15.63 - samples/sec: 1546.78 - lr: 0.000016\n",
            "2023-09-24 20:23:30,257 epoch 8 - iter 20/48 - loss 0.04985846 - time (sec): 20.01 - samples/sec: 1517.01 - lr: 0.000015\n",
            "2023-09-24 20:23:34,167 epoch 8 - iter 24/48 - loss 0.04881717 - time (sec): 23.92 - samples/sec: 1526.45 - lr: 0.000015\n",
            "2023-09-24 20:23:38,762 epoch 8 - iter 28/48 - loss 0.04843294 - time (sec): 28.51 - samples/sec: 1491.15 - lr: 0.000014\n",
            "2023-09-24 20:23:44,156 epoch 8 - iter 32/48 - loss 0.04763228 - time (sec): 33.91 - samples/sec: 1437.08 - lr: 0.000014\n",
            "2023-09-24 20:23:47,883 epoch 8 - iter 36/48 - loss 0.04814960 - time (sec): 37.64 - samples/sec: 1451.99 - lr: 0.000013\n",
            "2023-09-24 20:23:52,385 epoch 8 - iter 40/48 - loss 0.04781307 - time (sec): 42.14 - samples/sec: 1444.13 - lr: 0.000013\n",
            "2023-09-24 20:23:56,450 epoch 8 - iter 44/48 - loss 0.04832719 - time (sec): 46.20 - samples/sec: 1447.89 - lr: 0.000012\n",
            "2023-09-24 20:24:01,296 epoch 8 - iter 48/48 - loss 0.04859455 - time (sec): 51.05 - samples/sec: 1435.33 - lr: 0.000012\n",
            "2023-09-24 20:24:01,302 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:24:01,304 EPOCH 8 done: loss 0.0486 - lr 0.000012\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:24:04,368 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:24:04,384 DEV : loss 0.14900630712509155 - f1-score (micro avg)  0.652\n",
            "2023-09-24 20:24:04,393 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:24:09,323 epoch 9 - iter 4/48 - loss 0.04270381 - time (sec): 4.93 - samples/sec: 1244.67 - lr: 0.000011\n",
            "2023-09-24 20:24:13,881 epoch 9 - iter 8/48 - loss 0.04009955 - time (sec): 9.48 - samples/sec: 1278.54 - lr: 0.000011\n",
            "2023-09-24 20:24:18,484 epoch 9 - iter 12/48 - loss 0.03905215 - time (sec): 14.09 - samples/sec: 1297.24 - lr: 0.000011\n",
            "2023-09-24 20:24:22,425 epoch 9 - iter 16/48 - loss 0.04150960 - time (sec): 18.03 - samples/sec: 1347.26 - lr: 0.000010\n",
            "2023-09-24 20:24:26,810 epoch 9 - iter 20/48 - loss 0.04122218 - time (sec): 22.41 - samples/sec: 1355.97 - lr: 0.000010\n",
            "2023-09-24 20:24:31,461 epoch 9 - iter 24/48 - loss 0.04038085 - time (sec): 27.06 - samples/sec: 1345.98 - lr: 0.000009\n",
            "2023-09-24 20:24:34,938 epoch 9 - iter 28/48 - loss 0.04122336 - time (sec): 30.54 - samples/sec: 1387.77 - lr: 0.000009\n",
            "2023-09-24 20:24:39,178 epoch 9 - iter 32/48 - loss 0.03934472 - time (sec): 34.78 - samples/sec: 1387.98 - lr: 0.000008\n",
            "2023-09-24 20:24:43,613 epoch 9 - iter 36/48 - loss 0.03855040 - time (sec): 39.22 - samples/sec: 1396.23 - lr: 0.000008\n",
            "2023-09-24 20:24:47,640 epoch 9 - iter 40/48 - loss 0.03876447 - time (sec): 43.24 - samples/sec: 1406.90 - lr: 0.000007\n",
            "2023-09-24 20:24:52,202 epoch 9 - iter 44/48 - loss 0.03860718 - time (sec): 47.80 - samples/sec: 1404.05 - lr: 0.000007\n",
            "2023-09-24 20:24:55,738 epoch 9 - iter 48/48 - loss 0.03968363 - time (sec): 51.34 - samples/sec: 1427.16 - lr: 0.000006\n",
            "2023-09-24 20:24:55,742 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:24:55,744 EPOCH 9 done: loss 0.0397 - lr 0.000006\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:03<00:00,  1.75it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:24:59,196 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:24:59,215 DEV : loss 0.15591107308864594 - f1-score (micro avg)  0.6363\n",
            "2023-09-24 20:24:59,225 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:25:03,814 epoch 10 - iter 4/48 - loss 0.03614801 - time (sec): 4.58 - samples/sec: 1319.20 - lr: 0.000006\n",
            "2023-09-24 20:25:08,096 epoch 10 - iter 8/48 - loss 0.03623441 - time (sec): 8.87 - samples/sec: 1385.53 - lr: 0.000006\n",
            "2023-09-24 20:25:12,423 epoch 10 - iter 12/48 - loss 0.03396727 - time (sec): 13.19 - samples/sec: 1385.83 - lr: 0.000005\n",
            "2023-09-24 20:25:16,608 epoch 10 - iter 16/48 - loss 0.03459130 - time (sec): 17.38 - samples/sec: 1392.48 - lr: 0.000005\n",
            "2023-09-24 20:25:21,247 epoch 10 - iter 20/48 - loss 0.03434782 - time (sec): 22.02 - samples/sec: 1375.15 - lr: 0.000004\n",
            "2023-09-24 20:25:25,645 epoch 10 - iter 24/48 - loss 0.03273576 - time (sec): 26.42 - samples/sec: 1378.82 - lr: 0.000004\n",
            "2023-09-24 20:25:29,325 epoch 10 - iter 28/48 - loss 0.03367364 - time (sec): 30.10 - samples/sec: 1405.09 - lr: 0.000003\n",
            "2023-09-24 20:25:34,050 epoch 10 - iter 32/48 - loss 0.03310148 - time (sec): 34.82 - samples/sec: 1394.11 - lr: 0.000003\n",
            "2023-09-24 20:25:38,945 epoch 10 - iter 36/48 - loss 0.03255336 - time (sec): 39.71 - samples/sec: 1376.89 - lr: 0.000002\n",
            "2023-09-24 20:25:43,053 epoch 10 - iter 40/48 - loss 0.03432670 - time (sec): 43.82 - samples/sec: 1391.48 - lr: 0.000002\n",
            "2023-09-24 20:25:47,336 epoch 10 - iter 44/48 - loss 0.03462857 - time (sec): 48.11 - samples/sec: 1396.03 - lr: 0.000001\n",
            "2023-09-24 20:25:51,500 epoch 10 - iter 48/48 - loss 0.03528172 - time (sec): 52.27 - samples/sec: 1401.80 - lr: 0.000001\n",
            "2023-09-24 20:25:51,504 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:25:51,506 EPOCH 10 done: loss 0.0353 - lr 0.000001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:04<00:00,  1.49it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:25:55,550 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:25:55,578 DEV : loss 0.16027101874351501 - f1-score (micro avg)  0.6339\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:25:57,481 ----------------------------------------------------------------------------------------------------\n",
            "2023-09-24 20:25:57,489 Testing using last state of model ...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 6/6 [00:02<00:00,  2.04it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:26:00,455 Evaluating as a multi-label problem: False\n",
            "2023-09-24 20:26:00,476 0.5747\t0.6373\t0.6044\t0.4517\n",
            "2023-09-24 20:26:00,477 \n",
            "Results:\n",
            "- F-score (micro) 0.6044\n",
            "- F-score (macro) 0.5433\n",
            "- Accuracy 0.4517\n",
            "\n",
            "By class:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "      PERSON     0.7569    0.8209    0.7876       201\n",
            "         ORG     0.4713    0.4966    0.4837       149\n",
            "    LOCATION     0.5500    0.5946    0.5714        74\n",
            "        DATE     0.2838    0.3962    0.3307        53\n",
            "\n",
            "   micro avg     0.5747    0.6373    0.6044       477\n",
            "   macro avg     0.5155    0.5771    0.5433       477\n",
            "weighted avg     0.5830    0.6373    0.6084       477\n",
            "\n",
            "2023-09-24 20:26:00,482 ----------------------------------------------------------------------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_score': 0.6043737574552683,\n",
              " 'dev_score_history': [0.29352780309936183,\n",
              "  0.5638998682476944,\n",
              "  0.5924967658473479,\n",
              "  0.5986696230598669,\n",
              "  0.62582056892779,\n",
              "  0.6306098964326813,\n",
              "  0.6290672451193058,\n",
              "  0.6519823788546256,\n",
              "  0.6362694300518135,\n",
              "  0.6339381003201707],\n",
              " 'train_loss_history': [0.8916702703664834,\n",
              "  0.21932473314395443,\n",
              "  0.1523036105050467,\n",
              "  0.11594639986489899,\n",
              "  0.08728030946450224,\n",
              "  0.07104766294677414,\n",
              "  0.05734593832593048,\n",
              "  0.04859454562302461,\n",
              "  0.03968363003981193,\n",
              "  0.035281722130948816],\n",
              " 'dev_loss_history': [0.250569224357605,\n",
              "  0.15357275307178497,\n",
              "  0.13580676913261414,\n",
              "  0.13611148297786713,\n",
              "  0.13003411889076233,\n",
              "  0.1369745284318924,\n",
              "  0.1462002843618393,\n",
              "  0.14900630712509155,\n",
              "  0.15591107308864594,\n",
              "  0.16027101874351501]}"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "history = defaultdict(list)\n",
        "history['train_loss_history'] = [\n",
        "    0.8916702703664834,\n",
        "    0.21932473314395443,\n",
        "    0.1523036105050467,\n",
        "    0.11594639986489899,\n",
        "    0.08728030946450224,\n",
        "    0.07104766294677414,\n",
        "    0.05734593832593048,\n",
        "    0.04859454562302461,\n",
        "    0.03968363003981193,\n",
        "    0.035281722130948816\n",
        "]\n",
        "history['dev_loss_history'] = [\n",
        "    0.250569224357605,\n",
        "    0.15357275307178497,\n",
        "    0.13580676913261414,\n",
        "    0.13611148297786713,\n",
        "    0.13003411889076233,\n",
        "    0.1369745284318924,\n",
        "    0.1462002843618393,\n",
        "    0.14900630712509155,\n",
        "    0.15591107308864594,\n",
        "    0.16027101874351501\n",
        "  ]\n",
        "\n",
        "plt.figure()\n",
        "plt.plot(history['train_loss_history'], label='train loss')\n",
        "plt.plot(history['dev_loss_history'], label='val loss')\n",
        "plt.title('Training history')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()\n",
        "plt.ylim([0, 1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "id": "ANPeSOuQ2Ly7",
        "outputId": "e70bb753-02cf-4f59-ef96-c1fe5e7929e0"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.0, 1.0)"
            ]
          },
          "metadata": {},
          "execution_count": 21
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTOUlEQVR4nO3deXgTdf4H8HfuNG2T3ieFcp9tQS4BEZAqoIuCKIgox3qsLKBYcQUVEFBRFOSHKAjrvSAgiqJcYldUDhcEy31T2nK0pbRNeqZpMr8/0oSGHvRIO2nyfj1PnmYmM5lPqJg33/nMdySCIAggIiIichNSsQsgIiIiciaGGyIiInIrDDdERETkVhhuiIiIyK0w3BAREZFbYbghIiIit8JwQ0RERG6F4YaIiIjcCsMNERERuRWGGyKqk4kTJyI6OrpO+7722muQSCTOLaiGBg4ciC5dutxyu4sXL0IikeCzzz5r+KKIyKkYbojcjEQiqdFj165dYpfqlj788EMGIiKRSXhvKSL38p///Mdh+YsvvsDOnTvx5ZdfOqy/++67ERoaWufjmEwmWCwWqFSqWu9bWlqK0tJSqNXqOh+/rgYOHIisrCwcO3as2u0EQYDRaIRCoYBMJqvx+3fp0gVBQUEMj0QikotdABE512OPPeaw/Mcff2Dnzp0V1t+ssLAQGo2mxsdRKBR1qg8A5HI55HLX/t+PRCIRJXxVpri4GEqlElIpB9uJaoJ/U4g8kK3v5ODBg7jzzjuh0Wjw8ssvAwC+//573HfffYiIiIBKpULr1q2xYMECmM1mh/e4uefG1qPy7rvvYtWqVWjdujVUKhV69uyJAwcOOOxbWc+NRCLB1KlT8d1336FLly5QqVTo3Lkztm/fXqH+Xbt2oUePHlCr1WjdujU++uijWvfxnDhxAoMGDYJGo0FkZCQWLVrk8HplPTfp6emYNGkSmjVrBpVKhfDwcDzwwAO4ePEiACA6OhrHjx/Hr7/+aj/9N3DgQPv+Fy5cwMMPP4yAgABoNBrcfvvt2LJlS4XPJpFIsG7dOrz66quIjIyERqNBUlISJBIJ3nvvvQqfZe/evZBIJPjqq69q/PmJ3Jlr/9OJiBrM9evXMWzYMDzyyCN47LHH7KeoPvvsM/j4+CAhIQE+Pj7473//izlz5sBgMOCdd9655fuuXbsWeXl5+Mc//gGJRIJFixbhwQcfxIULF2452rN79258++23+Oc//wlfX18sW7YMo0aNQmpqKgIDAwEAf/31F4YOHYrw8HDMmzcPZrMZ8+fPR3BwcI0/e05ODoYOHYoHH3wQo0ePxsaNG/HSSy8hJiYGw4YNq3K/UaNG4fjx45g2bRqio6ORmZmJnTt3IjU1FdHR0Vi6dCmmTZsGHx8fvPLKKwBg/3PNyMhA3759UVhYiGeffRaBgYH4/PPPcf/992Pjxo0YOXKkw7EWLFgApVKJGTNmwGg0okOHDujXrx/WrFmD559/3mHbNWvWwNfXFw888ECN/wyI3JpARG5typQpws1/1QcMGCAAEFauXFlh+8LCwgrr/vGPfwgajUYoLi62r5swYYLQokUL+3JycrIAQAgMDBSys7Pt67///nsBgPDDDz/Y182dO7dCTQAEpVIpnDt3zr7u8OHDAgDh/ffft68bPny4oNFohMuXL9vXnT17VpDL5RXeszK2z/7FF1/Y1xmNRiEsLEwYNWpUhc/z6aefCoIgCDk5OQIA4Z133qn2/Tt37iwMGDCgwvrp06cLAITff//dvi4vL09o2bKlEB0dLZjNZkEQBOGXX34RAAitWrWq8Lv46KOPBADCyZMn7etKSkqEoKAgYcKECbf87ESegqeliDyUSqXCpEmTKqz38vKyP8/Ly0NWVhb69++PwsJCnDp16pbvO2bMGPj7+9uX+/fvD8B6SuZW4uPj0bp1a/tybGwstFqtfV+z2Yyff/4ZI0aMQEREhH27Nm3aVDvicjMfHx+HHiSlUolevXpVW6OXlxeUSiV27dqFnJycGh/LZuvWrejVqxfuuOMOhzqefvppXLx4ESdOnHDYfsKECQ6/CwAYPXo01Go11qxZY1+3Y8cOZGVl3bKnisiTMNwQeajIyEgolcoK648fP46RI0dCp9NBq9UiODjY/sWp1+tv+b7Nmzd3WLYFnZoEgpv3te1v2zczMxNFRUVo06ZNhe0qW1eVZs2aVejPKX+cyqhUKrz99tvYtm0bQkNDceedd2LRokVIT0+v0TFTUlLQvn37Cus7duxof728li1bVtjWz88Pw4cPx9q1a+3r1qxZg8jISNx11101qoPIEzDcEHmom0cFACA3NxcDBgzA4cOHMX/+fPzwww/YuXMn3n77bQCAxWK55ftWddm0UINZJ+qzb23U9TjTp0/HmTNnsHDhQqjVasyePRsdO3bEX3/95dT6gMp/PwAwfvx4XLhwAXv37kVeXh42b96MsWPH8koqonLYUExEdrt27cL169fx7bff4s4777SvT05OFrGqG0JCQqBWq3Hu3LkKr1W2riG0bt0aL7zwAl544QWcPXsWXbt2xeLFi+3zC1V1xVaLFi1w+vTpCuttp/patGhRo+MPHToUwcHBWLNmDXr37o3CwkI8/vjjdfw0RO6JUZ+I7GwjGuVHMEpKSvDhhx+KVZIDmUyG+Ph4fPfdd7hy5Yp9/blz57Bt27YGPXZhYSGKi4sd1rVu3Rq+vr4wGo32dd7e3sjNza2w/7333ov9+/dj37599nUFBQVYtWoVoqOj0alTpxrVIZfLMXbsWGzYsAGfffYZYmJiEBsbW7cPReSmOHJDRHZ9+/aFv78/JkyYgGeffRYSiQRffvml008L1cdrr72Gn376Cf369cPkyZNhNpuxfPlydOnSBUlJSQ123DNnzmDw4MEYPXo0OnXqBLlcjk2bNiEjIwOPPPKIfbvu3btjxYoVeP3119GmTRuEhITgrrvuwsyZM/HVV19h2LBhePbZZxEQEIDPP/8cycnJ+Oabb2p1Wmn8+PFYtmwZfvnlF/spQyK6geGGiOwCAwPx448/4oUXXsCrr74Kf39/PPbYYxg8eDCGDBkidnkArOFh27ZtmDFjBmbPno2oqCjMnz8fJ0+erNHVXHUVFRWFsWPHIjExEV9++SXkcjk6dOiADRs2YNSoUfbt5syZg5SUFCxatAh5eXkYMGAA7rrrLoSGhmLv3r146aWX8P7776O4uBixsbH44YcfcN9999Wqlu7du6Nz5844efIkxo0b5+yPStTk8d5SROQWRowYgePHj+Ps2bNil9IounXrhoCAACQmJopdCpHLYc8NETU5RUVFDstnz57F1q1bHW514M7+/PNPJCUlYfz48WKXQuSSOHJDRE1OeHg4Jk6ciFatWiElJQUrVqyA0WjEX3/9hbZt24pdXoM5duwYDh48iMWLFyMrKwsXLlxwmZt7ErkS9twQUZMzdOhQfPXVV0hPT4dKpUKfPn3w5ptvunWwAYCNGzdi/vz5aN++Pb766isGG6IqiDpy89tvv+Gdd97BwYMHcfXqVWzatAkjRoyodp9du3YhISEBx48fR1RUFF599VVMnDixUeolIiIi1ydqz01BQQHi4uLwwQcf1Gj75ORk3HfffRg0aBCSkpIwffp0PPnkk9ixY0cDV0pERERNhcv03EgkkluO3Lz00kvYsmULjh07Zl/3yCOPIDc3F9u3b2+EKomIiMjVNamem3379iE+Pt5h3ZAhQzB9+vQq9zEajQ6zh1osFmRnZyMwMLDKadKJiIjItQiCgLy8PERERNxy0ssmFW7S09MRGhrqsC40NBQGgwFFRUWV3mhu4cKFmDdvXmOVSERERA0oLS0NzZo1q3abJhVu6mLWrFlISEiwL+v1ejRv3hxpaWnQarUiVkZEREQ1ZTAYEBUVBV9f31tu26TCTVhYGDIyMhzWZWRkQKvVVjpqAwAqlQoqlarCeq1Wy3BDRETUxNSkpaRJzVDcp0+fClON79y5E3369BGpIiIiInI1ooab/Px8JCUl2e/km5ycjKSkJKSmpgKwnlIqP734M888gwsXLuBf//oXTp06hQ8//BAbNmzA888/L0b5RERE5IJEDTd//vknunXrhm7dugEAEhIS0K1bN8yZMwcAcPXqVXvQAYCWLVtiy5Yt2LlzJ+Li4rB48WL8+9//dpm7FRMREZH4XGaem8ZiMBig0+mg1+vZc0NE5KbMZjNMJpPYZVAtKZXKKi/zrs33d5NqKCYiIqqOIAhIT09Hbm6u2KVQHUilUrRs2RJKpbJe78NwQ0REbsMWbEJCQqDRaDhZaxNisVhw5coVXL16Fc2bN6/X747hhoiI3ILZbLYHm8DAQLHLoToIDg7GlStXUFpaCoVCUef3aVKXghMREVXF1mOj0WhEroTqynY6ymw21+t9GG6IiMit8FRU0+Ws3x3DDREREbkVhhsiIiI3Eh0djaVLl4r+HmJiQzEREZGIBg4ciK5duzotTBw4cADe3t5Oea+miuGGiIjIxQmCALPZDLn81l/bwcHBjVCRa+NpKSIiIpFMnDgRv/76K/7v//4PEokEEokEFy9exK5duyCRSLBt2zZ0794dKpUKu3fvxvnz5/HAAw8gNDQUPj4+6NmzJ37++WeH97z5lJJEIsG///1vjBw5EhqNBm3btsXmzZtrVWdqaioeeOAB+Pj4QKvVYvTo0cjIyLC/fvjwYQwaNAi+vr7QarXo3r07/vzzTwBASkoKhg8fDn9/f3h7e6Nz587YunVr3f/QaoAjN0RE5JYEQUCRqX6XFNeVl0JWoyt//u///g9nzpxBly5dMH/+fADWkZeLFy8CAGbOnIl3330XrVq1gr+/P9LS0nDvvffijTfegEqlwhdffIHhw4fj9OnTaN68eZXHmTdvHhYtWoR33nkH77//PsaNG4eUlBQEBATcskaLxWIPNr/++itKS0sxZcoUjBkzBrt27QIAjBs3Dt26dcOKFSsgk8mQlJRkn6dmypQpKCkpwW+//QZvb2+cOHECPj4+tzxufTDcEBGRWyoymdFpzg5Rjn1i/hBolLf+itXpdFAqldBoNAgLC6vw+vz583H33XfblwMCAhAXF2dfXrBgATZt2oTNmzdj6tSpVR5n4sSJGDt2LADgzTffxLJly7B//34MHTr0ljUmJibi6NGjSE5ORlRUFADgiy++QOfOnXHgwAH07NkTqampePHFF9GhQwcAQNu2be37p6amYtSoUYiJiQEAtGrV6pbHrC+eliIiInJRPXr0cFjOz8/HjBkz0LFjR/j5+cHHxwcnT55Eampqte8TGxtrf+7t7Q2tVovMzMwa1XDy5ElERUXZgw0AdOrUCX5+fjh58iQAICEhAU8++STi4+Px1ltv4fz58/Ztn332Wbz++uvo168f5s6diyNHjtTouPXBkRsiInJLXgoZTswfItqxneHmq55mzJiBnTt34t1330WbNm3g5eWFhx56CCUlJdW+z823MpBIJLBYLE6pEQBee+01PProo9iyZQu2bduGuXPnYt26dRg5ciSefPJJDBkyBFu2bMFPP/2EhQsXYvHixZg2bZrTjn8zhhsiInJLEomkRqeGxKZUKmt8u4E9e/Zg4sSJGDlyJADrSI6tP6ehdOzYEWlpaUhLS7OP3pw4cQK5ubno1KmTfbt27dqhXbt2eP755zF27Fh8+umn9jqjoqLwzDPP4JlnnsGsWbOwevXqBg03PC1FREQkoujoaPzvf//DxYsXkZWVVe2IStu2bfHtt98iKSkJhw8fxqOPPurUEZjKxMfHIyYmBuPGjcOhQ4ewf/9+jB8/HgMGDECPHj1QVFSEqVOnYteuXUhJScGePXtw4MABdOzYEQAwffp07NixA8nJyTh06BB++eUX+2sNheGGiIhIRDNmzIBMJkOnTp0QHBxcbf/MkiVL4O/vj759+2L48OEYMmQIbrvttgatTyKR4Pvvv4e/vz/uvPNOxMfHo1WrVli/fj0AQCaT4fr16xg/fjzatWuH0aNHY9iwYZg3bx4A600wp0yZgo4dO2Lo0KFo164dPvzww4atWRAEoUGP4GIMBgN0Oh30ej20Wq3Y5RARkZMUFxcjOTkZLVu2hFqtFrscqoPqfoe1+f7myA0RERG5FYYbIiIicisMN0RERORWGG6IiIjIrTDcEBERkVthuCEiIiK3wnBDREREboXhhoiIiNwKww0RERG5FYYbIiKiJi46OhpLly6t8vWJEydixIgRjVaP2BhuiIiIyK0w3BAREZFbYbghIiISyapVqxAREQGLxeKw/oEHHsDf//53AMD58+fxwAMPIDQ0FD4+PujZsyd+/vnneh3XaDTi2WefRUhICNRqNe644w4cOHDA/npOTg7GjRuH4OBgeHl5oW3btvj0008BACUlJZg6dSrCw8OhVqvRokULLFy4sF71OJtc7ALcjdkiQCaViF0GEREJAmAqFOfYCg0gufV3wcMPP4xp06bhl19+weDBgwEA2dnZ2L59O7Zu3QoAyM/Px7333os33ngDKpUKX3zxBYYPH47Tp0+jefPmdSrvX//6F7755ht8/vnnaNGiBRYtWoQhQ4bg3LlzCAgIwOzZs3HixAls27YNQUFBOHfuHIqKigAAy5Ytw+bNm7FhwwY0b94caWlpSEtLq1MdDYXhxkn2J2dj1rdHEOKrxldP3y52OUREZCoE3owQ59gvXwGU3rfczN/fH8OGDcPatWvt4Wbjxo0ICgrCoEGDAABxcXGIi4uz77NgwQJs2rQJmzdvxtSpU2tdWkFBAVasWIHPPvsMw4YNAwCsXr0aO3fuxMcff4wXX3wRqamp6NatG3r06AHA2rBsk5qairZt2+KOO+6ARCJBixYtal1DQ+NpKSfReSlw/loBjl7Ww2IRxC6HiIiaiHHjxuGbb76B0WgEAKxZswaPPPIIpFLrV3R+fj5mzJiBjh07ws/PDz4+Pjh58iRSU1PrdLzz58/DZDKhX79+9nUKhQK9evXCyZMnAQCTJ0/GunXr0LVrV/zrX//C3r177dtOnDgRSUlJaN++PZ599ln89NNPdf3oDYYjN07SOtgbXgoZ8o2lSL5egNbBPmKXRETk2RQa6wiKWMeuoeHDh0MQBGzZsgU9e/bE77//jvfee8/++owZM7Bz5068++67aNOmDby8vPDQQw+hpKSkISoHAAwbNgwpKSnYunUrdu7cicGDB2PKlCl49913cdtttyE5ORnbtm3Dzz//jNGjRyM+Ph4bN25ssHpqi+HGSeQyKTpFaHEwJQdHL+kZboiIxCaR1OjUkNjUajUefPBBrFmzBufOnUP79u1x22232V/fs2cPJk6ciJEjRwKwjuRcvHixzsdr3bo1lEol9uzZYz+lZDKZcODAAUyfPt2+XXBwMCZMmIAJEyagf//+ePHFF/Huu+8CALRaLcaMGYMxY8bgoYcewtChQ5GdnY2AgIA61+VMDDdOFBOps4aby3qM6BYpdjlERNREjBs3Dn/7299w/PhxPPbYYw6vtW3bFt9++y2GDx8OiUSC2bNnV7i6qja8vb0xefJkvPjiiwgICEDz5s2xaNEiFBYW4oknngAAzJkzB927d0fnzp1hNBrx448/omPHjgCAJUuWIDw8HN26dYNUKsXXX3+NsLAw+Pn51bkmZ2O4caKYSB0A4OglvciVEBFRU3LXXXchICAAp0+fxqOPPurw2pIlS/D3v/8dffv2RVBQEF566SUYDIZ6He+tt96CxWLB448/jry8PPTo0QM7duyAv78/AECpVGLWrFm4ePEivLy80L9/f6xbtw4A4Ovri0WLFuHs2bOQyWTo2bMntm7dau8RcgUSQRA8qvvVYDBAp9NBr9dDq9U69b3PZuTh7vd+g0Ypw9HXhvCScCKiRlRcXIzk5GS0bNkSarVa7HKoDqr7Hdbm+9t1YpYbaBXsA41ShsISM5Kz8sUuh4iIyCMx3DiRTCpB5whrmjzCU1NERESiYLhxsphIPwDA0csMN0RERGJguHGymGbWkRs2FRMREYmD4cbJbCM3x68YYOZMxUREjc7DrpNxK8763THcOFmrIG94K2UoMplx/hqbiomIGotCoQAAFBaKdLNMqjfbrMsymaxe78N5bpxMKpWgc6QO+5OzceSSHu1CfcUuiYjII8hkMvj5+SEzMxMAoNFoIKnBnbnJNVgsFly7dg0ajQZyef3iCcNNA4gtCzdHL+Xioe7NxC6HiMhjhIWFAYA94FDTIpVK0bx583qHUoabBhDTrGymYl4xRUTUqCQSCcLDwxESEgKTySR2OVRLSqXSKTMdM9w0ANttGE5cNaDUbIFcxtYmIqLGJJPJ6t23QU0Xv3UbQHSgN3xVchSbLDjHpmIiIqJGxXDTAKxNxZypmIiISAwMNw0ktpkfAE7mR0RE1NgYbhqIre+GTcVERESNi+GmgZRvKjaZLSJXQ0RE5DkYbhpIi0ANfNVylJRacDaDTcVERESNheGmgUgkknKnpnLFLYaIiMiDMNw0INtkfrxiioiIqPEw3DSg2LI7hB9jUzEREVGjYbhpQLbTUiev5qGklE3FREREjYHhpgFFBXhB56VAidmCMxl5YpdDRETkERhuGpBjUzFPTRERETUGhpsGxqZiIiKixsVw08Biy0Zu2FRMRETUOEQPNx988AGio6OhVqvRu3dv7N+/v9rtly5divbt28PLywtRUVF4/vnnUVxc3EjV1l6XsnBzKt0AY6lZ5GqIiIjcn6jhZv369UhISMDcuXNx6NAhxMXFYciQIcjMzKx0+7Vr12LmzJmYO3cuTp48iY8//hjr16/Hyy+/3MiV11wzfy/4axQwmQWcTmdTMRERUUMTNdwsWbIETz31FCZNmoROnTph5cqV0Gg0+OSTTyrdfu/evejXrx8effRRREdH45577sHYsWNvOdojJolEghjbHcJ5aoqIiKjBiRZuSkpKcPDgQcTHx98oRipFfHw89u3bV+k+ffv2xcGDB+1h5sKFC9i6dSvuvffeKo9jNBphMBgcHo0tJlILADjKpmIiIqIGJxfrwFlZWTCbzQgNDXVYHxoailOnTlW6z6OPPoqsrCzccccdEAQBpaWleOaZZ6o9LbVw4ULMmzfPqbXXVkzZTMUcuSEiImp4ojcU18auXbvw5ptv4sMPP8ShQ4fw7bffYsuWLViwYEGV+8yaNQt6vd7+SEtLa8SKrWyXg59Oz0OxiU3FREREDUm0kZugoCDIZDJkZGQ4rM/IyEBYWFil+8yePRuPP/44nnzySQBATEwMCgoK8PTTT+OVV16BVFoxq6lUKqhUKud/gFqI0KkR6K3E9YISnErPQ9coP1HrISIicmeijdwolUp0794diYmJ9nUWiwWJiYno06dPpfsUFhZWCDAymQwAIAhCwxVbT9amYs5UTERE1BhEPS2VkJCA1atX4/PPP8fJkycxefJkFBQUYNKkSQCA8ePHY9asWfbthw8fjhUrVmDdunVITk7Gzp07MXv2bAwfPtweclyV/TYMl3LFLYSIiMjNiXZaCgDGjBmDa9euYc6cOUhPT0fXrl2xfft2e5Nxamqqw0jNq6++ColEgldffRWXL19GcHAwhg8fjjfeeEOsj1BjtnDD2zAQERE1LIngyudzGoDBYIBOp4Ner4dWq220417VF6HPwv9CJpXg+LwhUCtce6SJiIjIldTm+7tJXS3VlIVp1QjyUcFsEXDiauPPtUNEROQpGG4aiUQiQWwz3kSTiIiooTHcNKIu7LshIiJqcAw3jSjWfsUUww0REVFDYbhpRLa5bs5m5qGohDMVExERNQSGm0YUqlUjxFcFiwCcuMrRGyIioobAcNPIbE3FPDVFRETUMBhuGpm9qZhXTBERETUIhptGxpEbIiKihsVw08hsIzfnr+WjwFgqcjVERETuh+GmkYX4qhGmVZc1FXOmYiIiImdjuBFBDE9NERERNRiGGxHY7hB+lE3FRERETsdwIwLbyM2RS7niFkJEROSGGG5EYBu5uZBVgHw2FRMRETkVw40IgnxUiNCpIQjAcZ6aIiIiciqGG5HYm4oZboiIiJyK4UYkbComIiJqGAw3Iolp5geAl4MTERE5G8ONSMo3FecVm0SuhoiIyH0w3IgkwFuJSD8vAMCxy5ypmIiIyFkYbkRkv4nm5VxxCyEiInIjDDci6mJvKubIDRERkbMw3IjIPnLDmYqJiIichuFGRLam4ovXC6EvYlMxERGRMzDciMhPo0RUgLWpmDMVExEROQfDjchiI/0AAEcYboiIiJyC4UZkXThTMRERkVMx3IjsRlMxww0REZEzMNyIrEuENdykZhcit7BE5GqIiIiaPoYbkek0CrQI1ADgTMVERETOwHDjAmyXhB/hTMVERET1xnDjAmx9N8fYVExERFRvDDcuwHbF1BE2FRMREdUbw40LsIWbSzlFyClgUzEREVF9MNy4AK1agZZB3gA43w0REVF9Mdy4iBhO5kdEROQUDDcugpP5EREROQfDjYvgbRiIiIicg+HGRXSO0EIiAS7nFuF6vlHscoiIiJoshhsX4cumYiIiIqdguHEhsZHsuyEiIqovhhsXEtPMDwBwhCM3REREdcZw40Jsl4PzNgxERER1x3DjQmxNxVf1xbiWx6ZiIiKiumC4cSHeKjlaB/sA4OgNERFRXTHcuJhY3kSTiIioXhhuXEyMbabiy7niFkJERNREMdy4GN5jioiIqH4YblxMpwgtpBIgw2BEpqFY7HKIiIiaHIYbF6NRytE2xBcAR2+IiIjqguHGBXVhUzEREVGdMdy4oNhm7LshIiKqK4YbF9SlXFOxIAgiV0NERNS0MNy4oE7hWsikElzLMyLDwJmKiYiIaoPhxgV5KWVoG2KdqfjIpVxxiyEiImpiGG5cFG+iSUREVDcMNy7K1lR8hOGGiIioVhhuXFSXciM3bComIiKqOYYbF9UxXAu5VIKs/BJc1XOmYiIioppiuHFRaoUM7UKtMxVzMj8iIqKaY7hxYWwqJiIiqj2GGxcWw6ZiIiKiWhM93HzwwQeIjo6GWq1G7969sX///mq3z83NxZQpUxAeHg6VSoV27dph69atjVRt44phUzEREVGtiRpu1q9fj4SEBMydOxeHDh1CXFwchgwZgszMzEq3Lykpwd13342LFy9i48aNOH36NFavXo3IyMhGrrxxdAj3hUImQXZBCS7nFoldDhERUZMgarhZsmQJnnrqKUyaNAmdOnXCypUrodFo8Mknn1S6/SeffILs7Gx899136NevH6KjozFgwADExcU1cuWNQyWXoX2Ytan4KJuKiYiIakS0cFNSUoKDBw8iPj7+RjFSKeLj47Fv375K99m8eTP69OmDKVOmIDQ0FF26dMGbb74Js9lc5XGMRiMMBoPDoymJieQdwomIiGpDtHCTlZUFs9mM0NBQh/WhoaFIT0+vdJ8LFy5g48aNMJvN2Lp1K2bPno3Fixfj9ddfr/I4CxcuhE6nsz+ioqKc+jkaWkykHwCGGyIiopoSvaG4NiwWC0JCQrBq1Sp0794dY8aMwSuvvIKVK1dWuc+sWbOg1+vtj7S0tEasuP7st2G4xKZiIiKimpCLdeCgoCDIZDJkZGQ4rM/IyEBYWFil+4SHh0OhUEAmk9nXdezYEenp6SgpKYFSqaywj0qlgkqlcm7xjahdqC+UMin0RSZcyilCVIBG7JKIiIhcmmgjN0qlEt27d0diYqJ9ncViQWJiIvr06VPpPv369cO5c+dgsVjs686cOYPw8PBKg407UMql6BDOmYqJiIhqStTTUgkJCVi9ejU+//xznDx5EpMnT0ZBQQEmTZoEABg/fjxmzZpl337y5MnIzs7Gc889hzNnzmDLli148803MWXKFLE+QqPowqZiIiKiGhPttBQAjBkzBteuXcOcOXOQnp6Orl27Yvv27fYm49TUVEilN/JXVFQUduzYgeeffx6xsbGIjIzEc889h5deekmsj9AoYiN1WAvg6OVcsUshIiJyeRLBw7pUDQYDdDod9Ho9tFqt2OXUyPErety3bDe0ajkOz70HEolE7JKIiIgaVW2+v5vU1VKeql2oL5RyKQzFpUjNLhS7HCIiIpfGcNMEKGRSdAy3plQ2FRMREVWP4aaJiIm0hptjbComIiKqFsNNExFbNlMxR26IiIiqx3DTRMSUzVR87LIeFotH9YATERHVCsNNE9E2xAcquRR5xlKksKmYiIioSgw3TYRcJkWnCFtTca64xRAREbkwhpsmJMY2UzH7boiIiKrEcNOExPA2DERERLfEcNOExDbzAwAcv2JgUzEREVEVGG6akNbB3lArpMg3liL5eoHY5RAREbkkhpsmRC6TonME+26IiIiqw3DTxNj6bjiZHxERUeUYbpoYW7jhbRiIiIgqx3DTxMTaZiq+ooeZTcVEREQVMNw0Ma2CfaBRylBYYkZyVr7Y5RAREbkchpsmRiaVoLN9pmKemiIiIroZw00TFMM7hBMREVWpTuEmLS0Nly5dsi/v378f06dPx6pVq5xWGFUtppl15IZNxURERBXVKdw8+uij+OWXXwAA6enpuPvuu7F//3688sormD9/vlMLpIpsIzfHrxjYVExERHSTOoWbY8eOoVevXgCADRs2oEuXLti7dy/WrFmDzz77zJn1USVaBXnDWylDkcmM89fYVExERFRencKNyWSCSqUCAPz888+4//77AQAdOnTA1atXnVcdVUoqlaAzJ/MjIiKqVJ3CTefOnbFy5Ur8/vvv2LlzJ4YOHQoAuHLlCgIDA51aIFUu1naH8Eu54hZCRETkYuoUbt5++2189NFHGDhwIMaOHYu4uDgAwObNm+2nq6hhxZRN5neUTcVEREQO5HXZaeDAgcjKyoLBYIC/v799/dNPPw2NRuO04qhqttswHL9iQKnZArmMV/UTEREBdRy5KSoqgtFotAeblJQULF26FKdPn0ZISIhTC6TKRQd6w0clh7HUgrOZbComIiKyqVO4eeCBB/DFF18AAHJzc9G7d28sXrwYI0aMwIoVK5xaIFVOKpWgS6R1vhuemiIiIrqhTuHm0KFD6N+/PwBg48aNCA0NRUpKCr744gssW7bMqQVS1WKb+QEAjvKKKSIiIrs6hZvCwkL4+voCAH766Sc8+OCDkEqluP3225GSkuLUAqlqXSLZVExERHSzOoWbNm3a4LvvvkNaWhp27NiBe+65BwCQmZkJrVbr1AKparbLwU9cNcBktohcDRERkWuoU7iZM2cOZsyYgejoaPTq1Qt9+vQBYB3F6datm1MLpKq1CNTAVy1HSakFZzLyxC6HiIjIJdQp3Dz00ENITU3Fn3/+iR07dtjXDx48GO+9957TiqPqSSQS+yXhvIkmERGRVZ0nRwkLC0O3bt1w5coV+x3Ce/XqhQ4dOjitOLo122R+vA0DERGRVZ3CjcViwfz586HT6dCiRQu0aNECfn5+WLBgASwW9n40Jo7cEBEROarTDMWvvPIKPv74Y7z11lvo168fAGD37t147bXXUFxcjDfeeMOpRVLVYiP9AAAnr+ahpNQCpZwzFRMRkWerU7j5/PPP8e9//9t+N3AAiI2NRWRkJP75z38y3DSiqAAv6LwU0BeZcCYjz355OBERkaeq0z/zs7OzK+2t6dChA7Kzs+tdFNVc+aZizndDRERUx3ATFxeH5cuXV1i/fPlyxMbG1rsoqh02FRMREd1Qp9NSixYtwn333Yeff/7ZPsfNvn37kJaWhq1btzq1QLq1GyM3ueIWQkRE5ALqNHIzYMAAnDlzBiNHjkRubi5yc3Px4IMP4vjx4/jyyy+dXSPdgi3cnE7Pg7HULHI1RERE4pIIgiA4680OHz6M2267DWaz637BGgwG6HQ66PV6t7lVhCAIuG3BTuQUmrB5aj/7DTWJiIjcRW2+v3ndsBuQSCS8iSYREVEZhhs3EVvWVHyUTcVEROThGG7chK3vhldMERGRp6vV1VIPPvhgta/n5ubWpxaqh5iyPpszGXkoNpmhVsjELYiIiEgktQo3Ol31s9/qdDqMHz++XgVR3UTo1Aj0VuJ6QQlOpeeha5Sf2CURERGJolbh5tNPP22oOqiebE3Fv565hqOX9Qw3RETksdhz40ZuNBXnilsIERGRiBhu3EgXNhUTEREx3LgT28jN2cx8FJtcdyJFIiKihsRw40bCtGoE+ahgtgg4cdUgdjlERESiYLhxIxKJBDGR1impOZkfERF5KoYbN2Ob74a3YSAiIk/FcONmYiN5GwYiIvJsDDduJsbeVJyHohI2FRMRkedhuHEzoVo1QnxVsAjAiascvSEiIs/DcOOGeBNNIiLyZAw3bsh2aopNxURE5IkYbtzQjdswMNwQEZHnYbhxQ7bbMJy/lo8CY6nI1RARETUuhhs3FOKrRphWXdZUzJmKiYjIszDcuCneRJOIiDwVw42bsvXdHGNTMREReRiXCDcffPABoqOjoVar0bt3b+zfv79G+61btw4SiQQjRoxo2AKbINsVU0cu5YpbCBERUSMTPdysX78eCQkJmDt3Lg4dOoS4uDgMGTIEmZmZ1e538eJFzJgxA/3792+kSpsW21w3F7IKkM+mYiIi8iCih5slS5bgqaeewqRJk9CpUyesXLkSGo0Gn3zySZX7mM1mjBs3DvPmzUOrVq0asdqmI8hHhQidGoIAHOepKSIi8iCihpuSkhIcPHgQ8fHx9nVSqRTx8fHYt29flfvNnz8fISEheOKJJ255DKPRCIPB4PDwFLamYk7mR0REnkTUcJOVlQWz2YzQ0FCH9aGhoUhPT690n927d+Pjjz/G6tWra3SMhQsXQqfT2R9RUVH1rrupiOVMxURE5IFEPy1VG3l5eXj88cexevVqBAUF1WifWbNmQa/X2x9paWkNXKXriGnmB4AzFRMRkWeRi3nwoKAgyGQyZGRkOKzPyMhAWFhYhe3Pnz+PixcvYvjw4fZ1FosFACCXy3H69Gm0bt3aYR+VSgWVStUA1bu+8k3FhmITtGqFyBURERE1PFFHbpRKJbp3747ExET7OovFgsTERPTp06fC9h06dMDRo0eRlJRkf9x///0YNGgQkpKSPOqUU00EeCsR6ecFADh+2XN6jYiIyLOJOnIDAAkJCZgwYQJ69OiBXr16YenSpSgoKMCkSZMAAOPHj0dkZCQWLlwItVqNLl26OOzv5+cHABXWk1VsMx0u5xbh6OVc9GkdKHY5REREDU70cDNmzBhcu3YNc+bMQXp6Orp27Yrt27fbm4xTU1MhlTap1iCX0iVSh23H0nGUIzdEROQhJIIgCGIX0ZgMBgN0Oh30ej20Wq3Y5TS4389ew+Mf70d0oAa7XhwkdjlERER1Upvvbw6JuLkuEdam4ovXC6EvMolcDRERUcNjuHFz/t5KRAXYmop5STgREbk/hhsPEBvpBwA4wnBDREQegOHGA9hvw8DJ/IiIyAMw3HgA3oaBiIg8CcONB7A1FadmFyK3sETkaoiIiBoWw40H0GkUaBGoAQAc43w3RETk5hhuPITtPlNHLueKWwgREVEDY7jxEDFsKiYiIg/BcOMhYthUTEREHoLhxkPYLge/lFOEnAI2FRMRkftiuPEQWrUCLYO8AXD0hoiI3BvDjQex990w3BARkRtjuPEg9iumLuWKWwgREVEDYrjxILamYs51Q0RE7ozhxoN0jtACAC7nFuF6vlHkaoiIiBoGw40H8VUr0CqYTcVEROTeGG48TCwn8yMiIjfHcONhuthvw8BwQ0RE7onhxsPENvMDABxjuCEiIjfFcONhOkdoIZEAV/XFyMwrFrscIiIip2O48TDeKjlaB/sA4OgNERG5J4YbD3SjqZjz3RARkfthuPFAXey3YcgVtxAiIqIGwHDjgWKb8R5TRETkvhhuPFCnCC2kEiDDYESGgU3FRETkXhhuPJBGKUebEGtTMSfzIyIid8Nw46FiIv0A8NQUERG5H4YbDxUTab2JJsMNERG5G4YbDxVTNlPx0ct6CIIgbjFEREROxHDjoTqFayGTSnAtz4gMg1HscoiIiJyG4cZDeSllaFvWVHzkUq64xRARETkRw40HiymbzI+3YSAiInfCcOPBYsom8zvCcENERG6E4caDxdjvMcWmYiIich8MNx6sY7gWcqkE1wtKcFXPmYqJiMg9MNx4MLVChrahvgCAI5ypmIiI3ATDjYeLZVMxERG5GYYbD8emYiIicjcMNx7uRlNxLpuKiYjILTDceLgO4b5QyCTIKTThcm6R2OUQERHVG8ONh1PJZWhX1lR8lE3FRETkBhhuCLFlfTe8QzgREbkDhhtCTKQfAIYbIiJyDww3ZG8qPsKZiomIyA0w3BDahflAKZNCX2TCpRw2FRMRUdPGcONMOSlAExz5UMllaB/GmYqJiMg9MNw4i+EqsGogsP4xoDBb7Gpq7cZkfrniFkJERFRPDDfOcukAYMwDTv0IrOgLXNgldkW1wtswEBGRu2C4cZZO9wNPJQJB7YC8q8AXI4CfZgOlJWJXViNd7DMVs6mYiIiaNoYbZwqPA57+Feg+CYAA7F0GfHw3kHVW7MpuqV2oL5RyKQzFpUjNLhS7HCIiojpjuHE2pQYYvhQYswbwCgCuJgEf3Qkc/Nylm42Vcik6sqmYiIjcAMNNQ+n4N2DyXqDlAMBUCPzwLLDhcZduNo7hTMVEROQGGG4akjYcePw74O75gFQBnPwBWNEPSP5N7MoqFWubqZgjN0RE1IQx3DQ0qRTo9xzw5M9AYBsg7wrw+f3Azrku12zcpdwVUxaL655CIyIiqg7DTWOJ6Ar84zfgtgkABGDPUuCTe4Dr50Uu7Ia2oT5QyaXIM5YihU3FRETURDHcNCalN3D/MmDMfwAvf+DKX8DK/sChL12i2Vghk6JThBYAcORSrrjFEBER1RHDjRg6Dgee2QNE9wdMBcDmqcDXE4CiHLErs99Ek303RETUVDHciEUXCYz/Hoh/DZDKgRPfW5uNL+4WtSz7HcJ5xRQRETVRDDdiksqAO54HntgJBLQGDJeBz/4GJM4HzCZRSoqL8gMA7E/OxuMf/w9/XnTdS9eJiIgqw3DjCiJvszYbd3scgAD8vhj4ZIgozcbtQn0xeWBryKUS/H42Cw+t3IdHV/+B/1243ui1EBER1YVE8LAbCRkMBuh0Ouj1emi1WrHLqej4d9YJ/4r1gNIHGLYI6PooIJE0ahlp2YX4cNc5fP3nJZSWXRZ+e6sAPDu4Lfq0CoSkkeshIiLPVpvvb4YbV6S/BHz7DyClrP+m80jgb+9Zr7BqZJdyCrFi13ls+DMNJrP1P5Ve0QF4Lr4t+rZmyCEiosZRm+9vlzgt9cEHHyA6OhpqtRq9e/fG/v37q9x29erV6N+/P/z9/eHv74/4+Phqt2+SdM2ACZuBwXOszcbHNwEr7gBS9jZ6Kc38NXhjZAx+fXEQHr+9BZQyKfZfzMa4f/8PD63ch9/OXONdxImIyKWIHm7Wr1+PhIQEzJ07F4cOHUJcXByGDBmCzMzMSrfftWsXxo4di19++QX79u1DVFQU7rnnHly+fLmRK29gUhnQ/wXgiZ+AgFaA4RLw2X3Af18Xpdk4ws8LC0Z0wW//GoSJfaOhlEtxMCUH4z/Zj5Ef7sUvpzMZcoiIyCWIflqqd+/e6NmzJ5YvXw4AsFgsiIqKwrRp0zBz5sxb7m82m+Hv74/ly5dj/Pjxt9y+SZyWupkxD9j2EpC0xroc2QMYtdoaekSSaSjGyl8vYM3/UmAstQAA4prp8OzgtrirQwhPVxERkVM1mdNSJSUlOHjwIOLj4+3rpFIp4uPjsW/fvhq9R2FhIUwmEwICAip93Wg0wmAwODyaHJUvMOJD4KFPAZUOuPyndWbjpK9Em9k4RKvGnOGd8PtLg/BU/5bwUshw+JIeT3z+J4Yv342fjqdzJIeIiEQharjJysqC2WxGaGiow/rQ0FCkp6fX6D1eeuklREREOASk8hYuXAidTmd/REVF1btu0XR5EJi8B2jeFyjJB757BvjmCaAoV7SSQnzVeOU+a8j5x4BW0ChlOHbZgKe/PIh7l+3G9mNXeRNOIiJqVKL33NTHW2+9hXXr1mHTpk1Qq9WVbjNr1izo9Xr7Iy0trZGrdDK/KGDij8CgVwGJDDj2jXUUJ6VmI10NJchHhVnDOmL3S3fhnwNbw1spw8mrBjzzn0O4d9nv2HKEIYeIiBqHqOEmKCgIMpkMGRkZDuszMjIQFhZW7b7vvvsu3nrrLfz000+IjY2tcjuVSgWtVuvwaPKkMmDAi8DfdwD+0YA+FfjsXuCXNwFzqailBXgr8a+hHbD7pbsw7a428FXJcSo9D1PWHsLQ//sNPxy+AjNDDhERNSBRw41SqUT37t2RmJhoX2exWJCYmIg+ffpUud+iRYuwYMECbN++HT169GiMUl1TVE/gH78DcWMBwQL8+jbw6TAg56LYlcHfW4kX7mmP3S/dhecGt4WvWo4zGfmY9tVfGLL0N3yfdJkhh4iIGoToV0utX78eEyZMwEcffYRevXph6dKl2LBhA06dOoXQ0FCMHz8ekZGRWLhwIQDg7bffxpw5c7B27Vr069fP/j4+Pj7w8fG55fGa5NVSNXF0I/Dj84DRACh9gfsWA3FjxK7KTl9kwud7L+Lj3cnQF1kvZW8V5I2pd7XB/XERkMua9BlSIiJqYE1uhuLly5fjnXfeQXp6Orp27Yply5ahd+/eAICBAwciOjoan332GQAgOjoaKSkpFd5j7ty5eO211255LLcNNwCQkwJ8+zSQ9od1OeZha8hR68Stq5y8YhO+2JeC1b9fQG6hNeREB2owZVAbjOwWyZBDRESVanLhpjG5dbgBrD03vy+2nqISzIBfc+DBfwPNe4tdmYN8Yym+2HcRq3+7gJyykNM8QIMpg1rjwduaQcGQQ0RE5TDcVMPtw41N2n7gmyeB3BTrVVUD/gX0nwHI5GJX5qDAWIr//JGCVb9dwPWCEgBAM38v/HNgGzzUvRmUcoYcIiJiuKmWx4QbACg2AFtnAEfWW5ejbgceXAX4txC3rkoUlpRi7f9SsfLXC8jKNwIAInRqTB7UBqN7NINKLhO5QiIiEhPDTTU8KtzYHNkA/JgAlOQBKi1w3xIg9mGxq6pUsclcFnLOIzPPGnLCdWpMHtgao3tEQa1gyCEi8kQMN9XwyHADWC8P/+Yp4FLZHdRjxwD3vguoXfPPoNhkxvoDaVix6zzSDcUAgFCtCs8MaI2xvZoz5BAReRiGm2p4bLgBrM3Gv70D/LbIOi+OXwtg1MfW+XJclLHUjA1/XsKKX87hit4acoJ9VfjHna0wrncLeCkZcoiIPAHDTTU8OtzYpP5hHcXRp1qbjQfOBPq/YJ352EUZS8345uBlfPDLOVzOLQIABPko8fSdrfDY7S2gUbpWozQRETkXw001GG7KFOUCW14Ajm20LjfvY2029msualm3UlJqwbeHLuGDXeeQlm0NOQHeSjzVvxXG92kBbxVDDhFRnVksgNkImIqAUiNQWlz286blW72uawbcPtmppTHcVIPhphxBsF5JtWVGWbOxDvjbEiDmIbEruyWT2YJNf1lHclKuFwIA/DUKPFkWcnzVCpErJCKqI4ulLCQUlwsMNy2bGuh1c4lzPkOzXsCTO53zXmUYbqrBcFOJ7GTg26eASwesy3GPAsPedtlm4/JKzRZ8n3QFy385h+SsAgCAzkuBJ+9oiQn9oqFlyCGihlZaApTkA8a8sp/51n8wGvNvLBvzKq6rsFxgHQFxVsCoL4kMUHgBchUgV5f9vHlZDSjUjstytfUsQM8nnFoOw001GG6qYDYBvy4Cfn/X2mwMCaCNsN513K+F9ad/ixvLPqGA1HUm2DNbBPxw+Are/+9ZnL9mDTlatRxjezXHgPbB6N7Cn3PlEJGVxVwuUFQWNPJu8fpNyw0ZRqTyG4Gh2kBRSfCoVTCpZBsXm/SV4aYaDDe3kLIX+O6fQE5y9dvZkrl/dMUA5NdCtFEfs0XAlqNX8X7iWZzNzLevVyuk6NUyEP3bBOGOtkHoEOYLiUQiSo1EBOtpcbMJsJisP8s/t5RaA0Olr5e9Vv65ucQ66lGjYJIHmAob5jPJvQCVD6D0KfvpW/2ySlvutbKHwutG6JCpXC5giInhphoMNzUgCEDBNeuNOHMuArkXrT9zUqwPw6Wy0Z1qeAVUHO2xBSFdM0DWsKeLLBYBP51Ix47jGdh9LgvXyiYEtAnyUaJfmyDcURZ2wnVeDVoPkehKjdYLCYpygOJc6/NifcWgYHtuMZULGKXVPC/brrLnlQUX23NLqch/ILCOiih9AJWvY8ioMpiUX+dbMZgwiDQohptqMNw4gdkE6NPKBZ6L1ntY2ZaLsqvfXyK1BhyH010tbyx7BwFOHFURBAGnM/Kw+2wWdp/Lwv8uZKPIZHbYpnWwN/q3DUa/NkG4vVUAG5LJNVnM1kBSlFMWTnIqBhaHZdt2uQ03WuFMUjkgVVj/8SNT3HgulQMyZfXPld5VBBPfSsJL2bJc5dT/11DDYripBsNNIyg2lAs7F28KQCnWywyro/CuZMSn3LJSU6/ySkotOJSag91ns/D7uSwcvZQLS7m/BTKpBN2i/NCvTRD6tw1CXJQf71JOziMI1lMj9jByUxBxCCrlX9MDRn09Dy4B1DrAyx/w8rOeFpGrywUFhTUsVPa8stBhf660jlpUu82t3kPBoEHVYripBsONyCwWID/jptGeizcCUN5VALf4T9I75EbYuTkAaSNrPRmhvtCEfResozq7z2bh4nXHf+H6qOS4vVVA2SmsYLQO9ma/Dlkvpa1shKRCSKnkNcFc9fvWhNIHUPvdCClefo7LDq/531hWaV3qQgCi2mC4qQbDjYszFZed8kqxNjWXP92VcxEwGqrfX6qwnvKyhR1NoPVfygDsoanS5RvPDcWluJJbiMu5RbiaUwhjqbW/SFK2jbdKhgidGpF+aoTpvKBRSGt9jKpfQyWvwfqFpPQBFJqy4feyh0JTdr5fYx3xUnpbn9u25bD7DYJgnXjMoem0uitk8qq5jDfPOidIfciUjsHDIYj4Vf2aWgfIlfX8wyBqemrz/c3uJ3ItCjUQ1Nb6uJkgWP/le3PgsS3nplkbFXOSb321VzW0ZY8OthU3/y0xA8gue7g6ieymIHRzMKokJCm9Kw9K5bdVaBpnBMBcWsUluDUJJTfvl1//EZObSaQ3TvPUNqQovBg8iRoIww01HRIJoAmwPiK6VXzdYgYMVxwDT7Gh7AtEcuM9rE8qfrFU+prjfqUW4Iq+GGnZhUjLKca1fCOs4y7W12USKcJ1akQFeqN5gAYhWjWkEmnF97f/qK6ucq+ZTdaG0JJC65e0qbDs0teyx83Ltr4mwWwd7brViFdd2AJPjYJR2XOprAZzh5RbV9/Rkaooy18Fc9OVL/bnVTSh2rbx8rc2rvI0D5HLYbgh9yGVAX5R1gf6N8gh5ACalz0A4Hq+EXvOX8eesiuxLucWAVmwPmCdLblv60Dc0TYI/dsEo3lg/Zqha8xcCpgKysJQQdnzsmVTQfXByFQWoKra18ZU2HhX4EgV5S7DrWLuEIcwUs02Cm8GEiI3x54bIicRBAHJWQXYcy4Lv5/Nwr7z15FndJzLIyrAC3e0CUb/tkHo2zoQfpom1jthsVinh69pGLKPMpU9t5RWMRpyi+AiV4n9yYlIZGworgbDDTWWUrMFRy7rrfPrnM3CodQclJa75lwiAWIiddarsNoEoXs0bxFBRFQVhptqMNyQWAqMpfhf8nX8fjYLe85l4UxGvsPrtltE3NEmEHe0CUaHMF9IpWw4JSICGG6qxXBDriLDUIzdZUHn9ypuEdG3tfX2EH1aBaKZvxfn1yEij8VwUw2GG3JFgiDgTEZ+2USC1/C/5GwUljhethzko0LXKB3imvkhLsoPcc38oNPwNhFE5BkYbqrBcENNQUmpBX+l5mB3WXPysct6h34dm1ZB3mVBR4e4KD90DNdCrWDfDhG5H4abajDcUFNUbDLj+BUDDqflIiktF4cv5SLlesXLsBUyCTqGa+2jO12jdGgV5MPeHSJq8hhuqsFwQ+4ip6AEhy/l4nCaHocvWUNPdkFJhe18VXLEljud1TXKD6FatQgVExHVHcNNNRhuyF0JgoBLOUXWkZ2y0Z2jl/UoNlkqbBumVSMuSmcPOzGROviq2b9DRK6L4aYaDDfkSUrNFpzJyLeO7KRaA8+ZjDzc3L4jkQBtgn2s/TtRfujazA/tw3yhlHMmXyJyDQw31WC4IU9XYCzFsct6+ymtpLRc620jbqKUS9ElQmsf3Ylr5ocWgRpejk5EomC4qQbDDVFF1/KM9lNZttNahuLSCtvpvBRlIzs6+yhPkA9vjUBEDY/hphoMN0S3JggCLl4vtF+dlZSWixNXDCgxV+zfaebvZT+VFRflhy6RWmiUvCcvETkXw001GG6I6qak1IJT6bbL0a2ntc5l5lfYTioB2oX6Wk9llZ3SahviA7mM/TtEVHcMN9VguCFyHkOxCUcv6e2nspLScpF5020kAMBLIUNMpA6dI7Vo5q9BpJ+X9eHvBX+Ngn08RHRLDDfVYLghaljp+mL7RINJqdbL0fONFft3bLwUMkT4qRFpDz1qRPp7IdJPgwg/NcK0ao76EBHDTXUYbogal9ki4MK1fCSl5eJsZj4u5xThcq71cfPNQisjk0oQplVbA1DZaE+EbeSnbJk9PkTurzbf3/w/AhE1KJlUgrahvmgb6lvhtWKTGen6YmvYKRd6LucU4Yq+CFdyi2AyC/b1B5BT6TH8NYobgcffMfhE+Hkh0FvJU19EHoThhohEo1bIEB3kjegg70pft1gEXMs3OoSfK+WDUE4R8oylyCk0IafQhONXDFUcR+o42uNXNvpTFoTCdGooeOqLyG0w3BCRy5JKJQjVqhGqVeO25v6VbmMoNlnDTtloz+WcIlwqF4Iy84woNllw4VoBLlwrqPw4EiBUq3YIPRF+XmhW7rmPiv+7JGoq+LeViJo0rVoBbbgCHcMrPwdvLC079XVT6LGNAl3JLUaJ2YKr+mJc1RcDKZWf+tJ5KW6EHz81QnVqhPpag1eIVoVQXzW0XnKe/iJyAQw3ROTWVHIZWgR6o0Vg1ae+smynvmz9PmXPL5U9NxSXQl9kgr7IhBNXKz/1ZT2WtGykSYUQrS38qKwByLdsnVYFHxVDEFFDYrghIo8mlUoQolUjRKtGtypOfeUVm27q9ylGZl4xMg1GZBiKkZlnhL7IBGOpBanZhUjNLqz2mBqlzB547GHIt2wEqOw0XKhWxavAiOqIf3OIiG7BV61AhzAFOoRVfflpsclsDTt5xcgwFCPDYESmodgefjIM1jCUZyxFYYkZyVkFSM6qvAfIflyV3CHwlB/9CS0bGQrRqqBWyJz9kYmaNIYbIiInUCtkaB6oQfNATbXbFRhL7WHHFngy86xhyBaE0vXFKDKZkWcsRd61UpyvohHaRuelsAeeYNtoUNlPWxgK9lVBJWcIIs/AcENE1Ii8VXK0VMnRsorL3wHrjUvzjaX20Z8bYcg6MpRpuBGGjKUWez/QmYyK9/oqL8BbeWP0x1eFEK0Kfl5KaL3k0HkprM3ZXgrrcy8FfFVySKXsDaKmh+GGiMjFSCQS+KoV8FUr0CbEp8rtBEGAoajUYeQno1wvkC0QXcszosRsQXZBCbILSnAqPa+GdVhPjdkDj9oWfKxhyBaCHIORdXutWsHTZSQahhsioiZKIpFAp1FAp1FUOgO0jSAIyC00lfUD2fp/inGtrBHadjWYoWwEyNYcLQiAobgUhuJSXMopqnV9Krm0XPiRO4Shm8OS1mGZo0ZUPww3RERuTiKRwN9bCX9vJTqE1WyfYpMZhmITDEVlwafYGn5sAchQXAp9oXW97XVrQCqFodgEQQCMpRZcyzPW6B5iFWu2jhrpNOVCTxUjR+WDke2hlHPGaU/GcENERBWoFTKoFTKEVD0gVCWLRUB+yU3hp6jUGo6Kb4wOOQSlcss3jxoBtR818lLIHELQzafQyq/XaRzXqxVSzkPUxDHcEBGRU0mlEmsPjlpRp/1rO2pkHzkqNCHPWApBAIpMZhSZzMgw1H7USCmT2vuHbj6V5jBipK4YkLyVMgYjF8BwQ0RELqU+o0Zmi4D8cjNKlw9ANz/KByZbaDJbBJSYLcjKNyIrv/bBSCaV2PuLKvQYVRKSyo8Y+arZZ+QsDDdEROQ2ZNIbTda1ZbsE3zYyVPH0WVUhyXrKrcRsgdki2O9SX1sSibUJWyWXQa2w/lTJpVArrD9VCinUchlUiorbqGzblD1Xl1unLvfajfdy3EYhc68eJYYbIiIiOF6CH+nnVat9BUFAsclSYWSostGiyoJSscnaZ1RsspS9TwN9yCrIpJIb4ah8cHIIVOVDVNUBTCWXIcRXhTvbBTfuhyiH4YaIiKieJBIJvJQyeCllCNOpa72/sdQMQ1Epik1mGEvNKDZZYCy1wGgyW3+WWn8W25ZN5Z7bt6+4TcXXbjwvKbXYj2+2CCgsMaOwxAyg9qNON+sa5cdwQ0RE5MlUchmCfRt30kNLWX+R0WRBcanZHoYcAlL5cOQQqBy3sb7HjTDWOrjqyScbA8MNERGRB5JKJVBLrc3bOtTtyjZX5V4dREREROTxGG6IiIjIrTDcEBERkVthuCEiIiK3wnBDREREboXhhoiIiNyKS4SbDz74ANHR0VCr1ejduzf2799f7fZff/01OnToALVajZiYGGzdurWRKiUiIiJXJ3q4Wb9+PRISEjB37lwcOnQIcXFxGDJkCDIzMyvdfu/evRg7diyeeOIJ/PXXXxgxYgRGjBiBY8eONXLlRERE5IokgiAIYhbQu3dv9OzZE8uXLwcAWCwWREVFYdq0aZg5c2aF7ceMGYOCggL8+OOP9nW33347unbtipUrV97yeAaDATqdDnq9Hlqt1nkfhIiIiBpMbb6/RR25KSkpwcGDBxEfH29fJ5VKER8fj3379lW6z759+xy2B4AhQ4ZUuT0RERF5FlFvv5CVlQWz2YzQ0FCH9aGhoTh16lSl+6Snp1e6fXp6eqXbG41GGI1G+7JerwdgTYBERETUNNi+t2tywsnt7y21cOFCzJs3r8L6qKgoEaohIiKi+sjLy4NOp6t2G1HDTVBQEGQyGTIyMhzWZ2RkICwsrNJ9wsLCarX9rFmzkJCQYF+2WCzIzs5GYGAgJBJJPT+BI4PBgKioKKSlpbGfxwXw9+Fa+PtwLfx9uB7+TqonCALy8vIQERFxy21FDTdKpRLdu3dHYmIiRowYAcAaPhITEzF16tRK9+nTpw8SExMxffp0+7qdO3eiT58+lW6vUqmgUqkc1vn5+Tmj/CpptVr+h+lC+PtwLfx9uBb+PlwPfydVu9WIjY3op6USEhIwYcIE9OjRA7169cLSpUtRUFCASZMmAQDGjx+PyMhILFy4EADw3HPPYcCAAVi8eDHuu+8+rFu3Dn/++SdWrVol5scgIiIiFyF6uBkzZgyuXbuGOXPmID09HV27dsX27dvtTcOpqamQSm9c1NW3b1+sXbsWr776Kl5++WW0bdsW3333Hbp06SLWRyAiIiIXInq4AYCpU6dWeRpq165dFdY9/PDDePjhhxu4qtpTqVSYO3duhdNgJA7+PlwLfx+uhb8P18PfifOIPokfERERkTOJfvsFIiIiImdiuCEiIiK3wnBDREREboXhhoiIiNwKw42TfPDBB4iOjoZarUbv3r2xf/9+sUvyWAsXLkTPnj3h6+uLkJAQjBgxAqdPnxa7LCrz1ltvQSKROEzESY3r8uXLeOyxxxAYGAgvLy/ExMTgzz//FLssj2Q2mzF79my0bNkSXl5eaN26NRYsWFCj+ydR1RhunGD9+vVISEjA3LlzcejQIcTFxWHIkCHIzMwUuzSP9Ouvv2LKlCn4448/sHPnTphMJtxzzz0oKCgQuzSPd+DAAXz00UeIjY0VuxSPlZOTg379+kGhUGDbtm04ceIEFi9eDH9/f7FL80hvv/02VqxYgeXLl+PkyZN4++23sWjRIrz//vtil9ak8VJwJ+jduzd69uyJ5cuXA7DeQiIqKgrTpk3DzJkzRa6Orl27hpCQEPz666+48847xS7HY+Xn5+O2227Dhx9+iNdffx1du3bF0qVLxS7L48ycORN79uzB77//LnYpBOBvf/sbQkND8fHHH9vXjRo1Cl5eXvjPf/4jYmVNG0du6qmkpAQHDx5EfHy8fZ1UKkV8fDz27dsnYmVko9frAQABAQEiV+LZpkyZgvvuu8/h7wo1vs2bN6NHjx54+OGHERISgm7dumH16tVil+Wx+vbti8TERJw5cwYAcPjwYezevRvDhg0TubKmzSVmKG7KsrKyYDab7beLsAkNDcWpU6dEqopsLBYLpk+fjn79+vEWHSJat24dDh06hAMHDohdise7cOECVqxYgYSEBLz88ss4cOAAnn32WSiVSkyYMEHs8jzOzJkzYTAY0KFDB8hkMpjNZrzxxhsYN26c2KU1aQw35NamTJmCY8eOYffu3WKX4rHS0tLw3HPPYefOnVCr1WKX4/EsFgt69OiBN998EwDQrVs3HDt2DCtXrmS4EcGGDRuwZs0arF27Fp07d0ZSUhKmT5+OiIgI/j7qgeGmnoKCgiCTyZCRkeGwPiMjA2FhYSJVRYD1nmU//vgjfvvtNzRr1kzscjzWwYMHkZmZidtuu82+zmw247fffsPy5cthNBohk8lErNCzhIeHo1OnTg7rOnbsiG+++Uakijzbiy++iJkzZ+KRRx4BAMTExCAlJQULFy5kuKkH9tzUk1KpRPfu3ZGYmGhfZ7FYkJiYiD59+ohYmecSBAFTp07Fpk2b8N///hctW7YUuySPNnjwYBw9ehRJSUn2R48ePTBu3DgkJSUx2DSyfv36VZga4cyZM2jRooVIFXm2wsJCSKWOX8UymQwWi0WkitwDR26cICEhARMmTECPHj3Qq1cvLF26FAUFBZg0aZLYpXmkKVOmYO3atfj+++/h6+uL9PR0AIBOp4OXl5fI1XkeX1/fCv1O3t7eCAwMZB+UCJ5//nn07dsXb775JkaPHo39+/dj1apVWLVqldileaThw4fjjTfeQPPmzdG5c2f89ddfWLJkCf7+97+LXVqTxkvBnWT58uV45513kJ6ejq5du2LZsmXo3bu32GV5JIlEUun6Tz/9FBMnTmzcYqhSAwcO5KXgIvrxxx8xa9YsnD17Fi1btkRCQgKeeuopscvySHl5eZg9ezY2bdqEzMxMREREYOzYsZgzZw6USqXY5TVZDDdERETkVthzQ0RERG6F4YaIiIjcCsMNERERuRWGGyIiInIrDDdERETkVhhuiIiIyK0w3BAREZFbYbghIo8nkUjw3XffiV0GETkJww0RiWrixImQSCQVHkOHDhW7NCJqonhvKSIS3dChQ/Hpp586rFOpVCJVQ0RNHUduiEh0KpUKYWFhDg9/f38A1lNGK1aswLBhw+Dl5YVWrVph48aNDvsfPXoUd911F7y8vBAYGIinn34a+fn5Dtt88skn6Ny5M1QqFcLDwzF16lSH17OysjBy5EhoNBq0bdsWmzdvbtgPTUQNhuGGiFze7NmzMWrUKBw+fBjjxo3DI488gpMnTwIACgoKMGTIEPj7++PAgQP4+uuv8fPPPzuElxUrVmDKlCl4+umncfToUWzevBlt2rRxOMa8efMwevRoHDlyBPfeey/GjRuH7OzsRv2cROQkAhGRiCZMmCDIZDLB29vb4fHGG28IgiAIAIRnnnnGYZ/evXsLkydPFgRBEFatWiX4+/sL+fn59te3bNkiSKVSIT09XRAEQYiIiBBeeeWVKmsAILz66qv25fz8fAGAsG3bNqd9TiJqPOy5ISLRDRo0CCtWrHBYFxAQYH/ep08fh9f69OmDpKQkAMDJkycRFxcHb29v++v9+vWDxWLB6dOnIZFIcOXKFQwePLjaGmJjY+3Pvb29odVqkZmZWdePREQiYrghItF5e3tXOE3kLF5eXjXaTqFQOCxLJBJYLJaGKImIGhh7bojI5f3xxx8Vljt27AgA6NixIw4fPoyCggL763v27IFUKkX79u3h6+uL6OhoJCYmNmrNRCQejtwQkeiMRiPS09Md1snlcgQFBQEAvv76a/To0QN33HEH1qxZg/379+Pjjz8GAIwbNw5z587FhAkT8Nprr+HatWuYNm0aHn/8cYSGhgIAXnvtNTzzzDMICQnBsGHDkJeXhz179mDatGmN+0GJqFEw3BCR6LZv347w8HCHde3bt8epU6cAWK9kWrduHf75z38iPDwcX331FTp16gQA0Gg02LFjB5577jn07NkTGo0Go0aNwpIlS+zvNWHCBBQXF+O9997DjBkzEBQUhIceeqjxPiARNSqJIAiC2EUQEVVFIpFg06ZNGDFihNilEFETwZ4bIiIicisMN0RERORW2HNDRC6NZ86JqLY4ckNERERuheGGiIiI3ArDDREREbkVhhsiIiJyKww3RERE5FYYboiIiMitMNwQERGRW2G4ISIiIrfCcENERERu5f8BQBPas05OIFQAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = corpus.test\n",
        "sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B_l723mGNGZ1",
        "outputId": "ef9d0f4f-575a-49fc-8ddb-d31550170691"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch.utils.data.dataset.ConcatDataset at 0x7ca51b742d40>"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# load tagger\n",
        "tagger: SequenceTagger = SequenceTagger.load('models/taggers/ner/202309242015/final-model.pt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "moehCNdINK7h",
        "outputId": "c3d030d9-011a-4075-f992-e41caccecec7"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:30:10,786 SequenceTagger predicts: Dictionary with 9 tags: O, B-PERSON, I-PERSON, B-ORG, I-ORG, B-LOCATION, I-LOCATION, B-DATE, I-DATE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_labels = get_test_labels(sentences)\n",
        "test_labels[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hid9lrs718RJ",
        "outputId": "03ffc4af-ec4a-4418-bf09-94477bde8c29"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pretty printing has been turned OFF\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'], ['B-PERSON', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'B-PERSON', 'B-ORG', 'I-ORG', 'O', 'O'], ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'B-PERSON', 'O', 'B-ORG', 'I-ORG', 'O', 'B-ORG', 'I-ORG', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOCATION', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'B-PERSON', 'B-PERSON', 'B-PERSON', 'B-PERSON']]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred_labels = get_pred_labels(sentences, tagger)\n",
        "pred_labels[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qdy8JLXW4zui",
        "outputId": "50b9cad3-67af-4d3b-e272-414dad3cc6d2"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Batch inference: 100%|██████████| 3/3 [00:02<00:00,  1.04it/s]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['B-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DATE', 'I-DATE', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DATE', 'O', 'B-PERSON', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'B-ORG', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'], ['B-PERSON', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'B-PERSON', 'O', 'O', 'O', 'B-PERSON', 'O'], ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'B-LOCATION', 'O', 'B-ORG', 'I-ORG', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOCATION', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'B-DATE', 'B-DATE', 'B-DATE', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'B-PERSON', 'B-PERSON', 'B-PERSON', 'B-PERSON']]"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "_test_labels = numpy.concatenate([numpy.array(i) for i in test_labels])\n",
        "_test_labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rr0wZxU_4zsf",
        "outputId": "3cfad713-23cf-4805-a2a5-d73d418ec844"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['O', 'O', 'O', ..., 'O', 'O', 'B-PERSON'], dtype='<U10')"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "_pred_labels = numpy.concatenate([numpy.array(i) for i in pred_labels])\n",
        "_pred_labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CrjPxRb14zqX",
        "outputId": "5a7c3702-7442-4cf5-cba5-5453eca8c713"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['B-PERSON', 'I-PERSON', 'O', ..., 'O', 'O', 'B-PERSON'],\n",
              "      dtype='<U10')"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "labels = ['B-DATE', 'B-LOCATION', 'B-ORG', 'B-PERSON', 'I-DATE', 'I-LOCATION', 'I-ORG', 'I-PERSON']\n",
        "print(classification_report(_test_labels, _pred_labels, labels=labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3mxQSQIO18PU",
        "outputId": "75bdaae6-7ab7-4eb4-b807-c07d73ea2bc9"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "      B-DATE       0.31      0.43      0.36        53\n",
            "  B-LOCATION       0.59      0.64      0.61        74\n",
            "       B-ORG       0.57      0.60      0.59       149\n",
            "    B-PERSON       0.77      0.84      0.80       201\n",
            "      I-DATE       0.46      0.39      0.42        46\n",
            "  I-LOCATION       0.62      0.60      0.61        30\n",
            "       I-ORG       0.48      0.54      0.51        70\n",
            "    I-PERSON       0.88      0.85      0.87        68\n",
            "\n",
            "   micro avg       0.62      0.67      0.64       691\n",
            "   macro avg       0.59      0.61      0.60       691\n",
            "weighted avg       0.63      0.67      0.64       691\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model Prediction**"
      ],
      "metadata": {
        "id": "AyPP0lTNP7Vx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load the trained model with best F1-score\n",
        "\n",
        "model = SequenceTagger.load('models/taggers/ner/202309242015/final-model.pt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "67482NNbFYro",
        "outputId": "cd64d858-8d3f-4fb9-add0-85028040baff"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-09-24 20:33:33,831 SequenceTagger predicts: Dictionary with 9 tags: O, B-PERSON, I-PERSON, B-ORG, I-ORG, B-LOCATION, I-LOCATION, B-DATE, I-DATE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from flair.data import Sentence\n",
        "\n",
        "def predict(text):\n",
        "    \"\"\"\n",
        "    Function to predict entities\n",
        "    \"\"\"\n",
        "    sentence = Sentence(text)\n",
        "\n",
        "    # predict tags and print\n",
        "    model.predict(sentence)\n",
        "\n",
        "    entities = []\n",
        "    for label in sentence.get_labels():\n",
        "        entity = dict()\n",
        "        entity[\"word\"] = label.data_point.text\n",
        "        entity[\"label\"] = label.value\n",
        "        entity[\"confidence\"] = float(\"{0:.5f}\".format(label.score))\n",
        "        entities.append(entity)\n",
        "\n",
        "    return entities"
      ],
      "metadata": {
        "id": "XnAmQxDC533f"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check sample test emails\n",
        "\n",
        "test = pd.read_csv('data/ner/test.csv')\n",
        "test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "s8mqvYqn5qrI",
        "outputId": "e38ce2f9-a711-4e0a-9c25-821a1a366901"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                       message_id  \\\n",
              "0   <15886210.1075854965508.JavaMail.evans@thyme>   \n",
              "1   <17217905.1075853978006.JavaMail.evans@thyme>   \n",
              "2   <28372483.1075843343800.JavaMail.evans@thyme>   \n",
              "3    <7553175.1075863444700.JavaMail.evans@thyme>   \n",
              "4   <16687289.1075862330552.JavaMail.evans@thyme>   \n",
              "..                                            ...   \n",
              "91  <17780105.1075840540168.JavaMail.evans@thyme>   \n",
              "92  <16492427.1075847631885.JavaMail.evans@thyme>   \n",
              "93  <11901989.1075844026014.JavaMail.evans@thyme>   \n",
              "94  <17922592.1075839996237.JavaMail.evans@thyme>   \n",
              "95  <26340375.1075840808386.JavaMail.evans@thyme>   \n",
              "\n",
              "                                                 text  \\\n",
              "0   Allright fellas here's the gameplan . . . I'll...   \n",
              "1   Tina,\\nKoch never returned my calls. Based on ...   \n",
              "2   I wanted to follow-up with everyone following ...   \n",
              "3   Vince,\\n\\n  I agree with you that it's a lesso...   \n",
              "4   I trust everything is going well.  Please let ...   \n",
              "..                                                ...   \n",
              "91  Heidi,\\n\\nI have no way of being able to discu...   \n",
              "92  Teh rest of the parts for Phil's PC arrived to...   \n",
              "93  Hey guys thanks a million for supporting the g...   \n",
              "94  Keoni,\\nI am not concerned with the improperly...   \n",
              "95  I will be circulating in the next few minutes ...   \n",
              "\n",
              "                                             entities  \\\n",
              "0   [{'word': 'Kevin A. Boone', 'tag': 'PERSON'}, ...   \n",
              "1   [{'word': 'Tina', 'tag': 'PERSON'}, {'word': '...   \n",
              "2   [{'word': 'Steve M', 'tag': 'PERSON'}, {'word'...   \n",
              "3   [{'word': 'Vince', 'tag': 'PERSON'}, {'word': ...   \n",
              "4   [{'word': 'next Board meeting', 'tag': 'DATE'}...   \n",
              "..                                                ...   \n",
              "91  [{'word': 'Heidi', 'tag': 'PERSON'}, {'word': ...   \n",
              "92  [{'word': 'Phil', 'tag': 'PERSON'}, {'word': '...   \n",
              "93  [{'word': 'Next Wednesday August 16th', 'tag':...   \n",
              "94  [{'word': 'Keoni', 'tag': 'PERSON'}, {'word': ...   \n",
              "95  [{'word': 'Cris Sherman', 'tag': 'PERSON'}, {'...   \n",
              "\n",
              "                                           annotation  \n",
              "0   [('Kevin A. Boone', 'PERSON'), ('Matt', 'PERSO...  \n",
              "1   [('Tina', 'PERSON'), ('Koch', 'PERSON'), ('Cyn...  \n",
              "2   [('Steve M', 'PERSON'), ('Janine', 'PERSON'), ...  \n",
              "3   [('Vince', 'PERSON'), ('Frank', 'PERSON'), ('S...  \n",
              "4   [('next Board meeting', 'DATE'), ('2001', 'DAT...  \n",
              "..                                                ...  \n",
              "91  [('Heidi', 'PERSON'), ('Randall Chapman', 'PER...  \n",
              "92  [('Phil', 'PERSON'), ('Kathy', 'PERSON'), ('Al...  \n",
              "93  [('Next Wednesday August 16th', 'DATE'), ('41s...  \n",
              "94  [('Keoni', 'PERSON'), ('Bill', 'PERSON'), ('CA...  \n",
              "95  [('Cris Sherman', 'PERSON'), ('Theresa Vos', '...  \n",
              "\n",
              "[96 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e06951c0-9fa5-451d-8943-958a54729291\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message_id</th>\n",
              "      <th>text</th>\n",
              "      <th>entities</th>\n",
              "      <th>annotation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>&lt;15886210.1075854965508.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Allright fellas here's the gameplan . . . I'll...</td>\n",
              "      <td>[{'word': 'Kevin A. Boone', 'tag': 'PERSON'}, ...</td>\n",
              "      <td>[('Kevin A. Boone', 'PERSON'), ('Matt', 'PERSO...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;17217905.1075853978006.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Tina,\\nKoch never returned my calls. Based on ...</td>\n",
              "      <td>[{'word': 'Tina', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Tina', 'PERSON'), ('Koch', 'PERSON'), ('Cyn...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>&lt;28372483.1075843343800.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I wanted to follow-up with everyone following ...</td>\n",
              "      <td>[{'word': 'Steve M', 'tag': 'PERSON'}, {'word'...</td>\n",
              "      <td>[('Steve M', 'PERSON'), ('Janine', 'PERSON'), ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>&lt;7553175.1075863444700.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Vince,\\n\\n  I agree with you that it's a lesso...</td>\n",
              "      <td>[{'word': 'Vince', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Vince', 'PERSON'), ('Frank', 'PERSON'), ('S...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>&lt;16687289.1075862330552.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I trust everything is going well.  Please let ...</td>\n",
              "      <td>[{'word': 'next Board meeting', 'tag': 'DATE'}...</td>\n",
              "      <td>[('next Board meeting', 'DATE'), ('2001', 'DAT...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>&lt;17780105.1075840540168.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Heidi,\\n\\nI have no way of being able to discu...</td>\n",
              "      <td>[{'word': 'Heidi', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Heidi', 'PERSON'), ('Randall Chapman', 'PER...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>&lt;16492427.1075847631885.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Teh rest of the parts for Phil's PC arrived to...</td>\n",
              "      <td>[{'word': 'Phil', 'tag': 'PERSON'}, {'word': '...</td>\n",
              "      <td>[('Phil', 'PERSON'), ('Kathy', 'PERSON'), ('Al...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>&lt;11901989.1075844026014.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Hey guys thanks a million for supporting the g...</td>\n",
              "      <td>[{'word': 'Next Wednesday August 16th', 'tag':...</td>\n",
              "      <td>[('Next Wednesday August 16th', 'DATE'), ('41s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>&lt;17922592.1075839996237.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>Keoni,\\nI am not concerned with the improperly...</td>\n",
              "      <td>[{'word': 'Keoni', 'tag': 'PERSON'}, {'word': ...</td>\n",
              "      <td>[('Keoni', 'PERSON'), ('Bill', 'PERSON'), ('CA...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>&lt;26340375.1075840808386.JavaMail.evans@thyme&gt;</td>\n",
              "      <td>I will be circulating in the next few minutes ...</td>\n",
              "      <td>[{'word': 'Cris Sherman', 'tag': 'PERSON'}, {'...</td>\n",
              "      <td>[('Cris Sherman', 'PERSON'), ('Theresa Vos', '...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>96 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e06951c0-9fa5-451d-8943-958a54729291')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e06951c0-9fa5-451d-8943-958a54729291 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e06951c0-9fa5-451d-8943-958a54729291');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-09dadadd-7476-4c81-b72b-0badd94750ec\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-09dadadd-7476-4c81-b72b-0badd94750ec')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-09dadadd-7476-4c81-b72b-0badd94750ec button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_email = test['text'].iloc[0]\n",
        "print(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b6Fhw0Cz5tN1",
        "outputId": "a4e13054-814f-4630-9843-fd7377e55f47"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Allright fellas here's the gameplan . . . I'll drive . . . leave at 8am\n",
            "tomorrow morning . . . we can meet at my house or at Matt's it doesn't\n",
            "really matter to me . . . plan on leaving BR no later than 9am sunday morn\n",
            ". . . Geaux Tigers\n",
            "\n",
            "Regards,\n",
            "\n",
            "Kevin A. Boone\n",
            "Accenture  Houston\n",
            "Consultant  Energy\n",
            "(Octel) 713.837.1638\n",
            "(Client) 281.848.1619\n",
            "(C) 713.306.7940\n",
            "(H) 713.864.4149\n",
            "Kevin.A.Boone@accenture.com\n",
            "\n",
            "Our web address is http://www.accenture.com\n",
            "\n",
            "\n",
            "This message is for the designated recipient only and may contain\n",
            "privileged or confidential information.  If you have received it in error,\n",
            "please notify the sender immediately and delete the original.  Any other\n",
            "use of the email by you is prohibited.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%pprint\n",
        "predict(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lSlT3Z836dok",
        "outputId": "958cc31e-d100-41ac-d714-fcc2600968e8"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'word': 'tomorrow morning', 'label': 'DATE', 'confidence': 0.8091},\n",
              " {'word': 'Matt', 'label': 'PERSON', 'confidence': 0.98989},\n",
              " {'word': 'Accenture', 'label': 'ORG', 'confidence': 0.979},\n",
              " {'word': 'Houston', 'label': 'LOCATION', 'confidence': 0.48811}]"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_email = test['text'].iloc[1]\n",
        "print(sample_email)"
      ],
      "metadata": {
        "id": "Psgt_T_dQX_e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3674baef-c736-4cfe-ed8a-2c40b0bd5c7c"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tina,\n",
            "Koch never returned my calls. Based on the contractual information, we may be \n",
            "putting ourselves in a risk area by 'assigning' the deals without proper \n",
            "documentation. The documentation that was supplied did not refence any ENA \n",
            "agreements, nor any specific meter numbers. If the purchase was indeed a \n",
            "meter/well sale; there should be documentation listing the meters acquired \n",
            "and the effective date. Since we are apparently the supplier, the risk is not \n",
            "as substaintial as if we were the purchaser. However, without documentation \n",
            "we are basically relying on the relationship btw Koch & Duke.\n",
            "\n",
            "Cyndie\n",
            "ENA Global Contracts\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Tina Valadez@ECT\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict(sample_email)"
      ],
      "metadata": {
        "id": "csMrldo0QX9d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c22b81b8-4626-4b8e-8ff8-8207d287d8db"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'word': 'Tina', 'label': 'PERSON', 'confidence': 0.95944},\n",
              " {'word': 'Koch', 'label': 'PERSON', 'confidence': 0.73476},\n",
              " {'word': 'ENA', 'label': 'ORG', 'confidence': 0.92136},\n",
              " {'word': 'Cyndie', 'label': 'PERSON', 'confidence': 0.61575},\n",
              " {'word': 'ENA Global', 'label': 'ORG', 'confidence': 0.80697},\n",
              " {'word': 'Tina Valadez', 'label': 'PERSON', 'confidence': 0.99369},\n",
              " {'word': 'ECT', 'label': 'ORG', 'confidence': 0.96524}]"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_email = test['text'].iloc[2]\n",
        "print(sample_email)"
      ],
      "metadata": {
        "id": "Z8x3uCNdQX7X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f556635-abe4-4e15-8ced-acc82b138fd5"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I wanted to follow-up with everyone following yesterday's meeting. \n",
            "\n",
            "It appears to me that we need to develop (1) a better analysis of the four \n",
            "market models - AGL (Steve M), Columbia of Ohio (Janine), Socal Gas (Jeff), \n",
            "NICOR (Roy) - based on some key elements and (2) the key Influence parties in \n",
            "Illinois with 5 layer Influence Circles that we need to be thinking about in \n",
            "this discussion.\n",
            "\n",
            "It would be great if we could get this to Laura for distribution by end of \n",
            "day Friday (realize it's tight).\n",
            "\n",
            "Key elements for Market Models \n",
            "\n",
            "1. Direct Access Allowed?  For What customers?  What timeline?\n",
            "2. Upstream Capacity resolution - assignment or otherwise?  Are there assets \n",
            "left to optimize\n",
            "3. Retail pricing - Fixed price vs. PBR (how?) vs. something else?\n",
            "4. Role of Wholesale Outsourcing Agent (would ENA sell to marketers or \n",
            "utilities or both?)\n",
            "5. Other Issues\n",
            "\n",
            "Thanks,\n",
            "\n",
            "Jim\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eb9yKw0Y9KYo",
        "outputId": "e5b7d847-0de2-4156-9506-d2df29b4ee67"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'word': 'yesterday', 'label': 'DATE', 'confidence': 0.70943},\n",
              " {'word': 'Steve', 'label': 'PERSON', 'confidence': 0.61155},\n",
              " {'word': 'of', 'label': 'ORG', 'confidence': 0.85975},\n",
              " {'word': 'Janine', 'label': 'PERSON', 'confidence': 0.95581},\n",
              " {'word': 'Socal Gas', 'label': 'ORG', 'confidence': 0.80789},\n",
              " {'word': 'Jeff', 'label': 'PERSON', 'confidence': 0.99384},\n",
              " {'word': 'NICOR', 'label': 'PERSON', 'confidence': 0.96538},\n",
              " {'word': 'Roy', 'label': 'PERSON', 'confidence': 0.99177},\n",
              " {'word': 'Illinois', 'label': 'LOCATION', 'confidence': 0.97919},\n",
              " {'word': 'Laura', 'label': 'PERSON', 'confidence': 0.99627},\n",
              " {'word': 'end', 'label': 'DATE', 'confidence': 0.62944},\n",
              " {'word': 'Friday', 'label': 'DATE', 'confidence': 0.64875},\n",
              " {'word': 'ENA', 'label': 'ORG', 'confidence': 0.88702},\n",
              " {'word': 'Jim', 'label': 'PERSON', 'confidence': 0.99733}]"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_email = test['text'].iloc[3]\n",
        "print(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h52auwCi9KVz",
        "outputId": "2e1dc0e3-6444-4173-fd16-b74ab030cbee"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Vince,\n",
            "\n",
            "  I agree with you that it's a lesson people need to learn\n",
            "over and over again.  I can't tell you how many politicians\n",
            "I met over the past year who really don't like markets and\n",
            "certainly don't understand how or why they work.  These\n",
            "aren't just the minor leaguers in Sacramento, but the big\n",
            "league players in Washington.\n",
            "\n",
            "  I also agree that the academic community can play \"an important\n",
            "role in shaping public opinion and in explaining the logic of\n",
            "deregulation process.\"  I'd like to think that is in large\n",
            "part what I have been trying to do.\n",
            "\n",
            "Frank\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oHxqbP3s9KTd",
        "outputId": "642d67bc-170e-43f1-fb7c-d8e863209dcf"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'word': 'Vince', 'label': 'PERSON', 'confidence': 0.99705},\n",
              " {'word': 'Sacramento', 'label': 'LOCATION', 'confidence': 0.98591},\n",
              " {'word': 'Washington', 'label': 'LOCATION', 'confidence': 0.8984},\n",
              " {'word': 'Frank', 'label': 'PERSON', 'confidence': 0.9975}]"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_email = test['text'].iloc[4]\n",
        "print(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gO-Uo0Ep8l9o",
        "outputId": "621921ca-8849-4a47-cccd-bdeb5e3ab69a"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I trust everything is going well.  Please let me know if there is anything I can do.  At the next Board meeting I am sheduled to provide the Board with an update of Enron Global Services.  Could you have someone prepare a couple of slides on Portland.  One should deal with the major accomplishements of your group in 2001 (ie rate case etc) in bullet form.  The second should be an update of the deal with NNG.  Perhaps a summary of the deal and then where we are in the timeline of key events.  Call if you have questions.  Thanks.  The Board meeting is scheduled for December 6th.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KiqOr2aT8l8e",
        "outputId": "b7144e16-8f65-4dfd-a825-e0369d1c504e"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'word': 'Enron Global Services', 'label': 'ORG', 'confidence': 0.94899},\n",
              " {'word': 'Portland', 'label': 'LOCATION', 'confidence': 0.96923},\n",
              " {'word': '2001', 'label': 'DATE', 'confidence': 0.59624},\n",
              " {'word': 'NNG', 'label': 'ORG', 'confidence': 0.43358},\n",
              " {'word': 'December 6th', 'label': 'DATE', 'confidence': 0.74267}]"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_email = test['text'].iloc[5]\n",
        "print(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IezK-xxz8l7E",
        "outputId": "dd6ea06f-1aa2-4119-e248-ecfbd4c38700"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Good morning Richard:\n",
            "\n",
            "Your resume was forwarded to Vince Kaminski and the Research group\n",
            "and they would like to conduct a telephone interview with you at your\n",
            "convenience.\n",
            "\n",
            "Please give me some dates and times that you would be available and\n",
            "I will coordinate the schedule.  Also, the telephone number you wish to be\n",
            "contacted at.  \n",
            "\n",
            "The telephone interview will be last approximately an hour and the \n",
            "interviewers would be:\n",
            "\n",
            "Vince Kaminski\t\tManaging Director, Research\n",
            "Stinson Gibner\t\tVice President, Research\n",
            "Vasant Shanbhogue\tVice President, Research\n",
            "\n",
            "Thanks Richard and we look forward to hearing from you.\n",
            "\n",
            "Regards,\n",
            "\n",
            "Shirley Crenshaw\n",
            "Administrative Coordinator\n",
            "Enron Research Group\n",
            "713-853-5290\n",
            "email:  Shirley.Crenshaw@enron.com\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict(sample_email)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x5E2RHts8l3j",
        "outputId": "e2383945-7548-43e9-fdc3-1f71ed666fe2"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'word': 'Richard', 'label': 'PERSON', 'confidence': 0.88917},\n",
              " {'word': 'Vince Kaminski', 'label': 'PERSON', 'confidence': 0.94147},\n",
              " {'word': 'Vince Kaminski', 'label': 'PERSON', 'confidence': 0.94577},\n",
              " {'word': 'Stinson Gibner', 'label': 'PERSON', 'confidence': 0.9046},\n",
              " {'word': 'Richard', 'label': 'PERSON', 'confidence': 0.93722},\n",
              " {'word': 'Shirley Crenshaw', 'label': 'PERSON', 'confidence': 0.94959},\n",
              " {'word': 'Enron Research Group', 'label': 'ORG', 'confidence': 0.92532}]"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zZqDT3U08l1f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "o_9nAzRd8lzm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2S8lpo5w8lxc"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}