MLP_Bert_tokenizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6G_h1TsofrLE",
        "outputId": "1b1f5a78-c722-42c4-b1aa-fcf071fb14c6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tensorflow==2.8.*\n",
            "  Downloading tensorflow-2.8.4-cp37-cp37m-manylinux2010_x86_64.whl (497.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 497.9 MB 4.6 kB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.1.2)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.21.6)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (0.2.0)\n",
            "Collecting tensorflow-estimator<2.9,>=2.8\n",
            "  Downloading tensorflow_estimator-2.8.0-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 63.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.15.0)\n",
            "Collecting tensorboard<2.9,>=2.8\n",
            "  Downloading tensorboard-2.8.0-py3-none-any.whl (5.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.8 MB 61.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.6.3)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (3.1.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (3.3.0)\n",
            "Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.12)\n",
            "Collecting keras<2.9,>=2.8.0rc0\n",
            "  Downloading keras-2.8.0-py2.py3-none-any.whl (1.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4 MB 51.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (0.4.0)\n",
            "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (14.0.6)\n",
            "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (3.19.6)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (4.1.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (0.27.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (57.4.0)\n",
            "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.3.0)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.14.1)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (2.1.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8.*) (1.50.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow==2.8.*) (0.38.4)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow==2.8.*) (1.5.2)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (0.6.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (1.8.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (3.4.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (2.23.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (2.14.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8.*) (0.4.6)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (4.9)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (5.2.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (4.13.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (3.10.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (2.10)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow==2.8.*) (3.2.2)\n",
            "Installing collected packages: tensorflow-estimator, tensorboard, keras, tensorflow\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.9.0\n",
            "    Uninstalling tensorflow-estimator-2.9.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.9.0\n",
            "  Attempting uninstall: tensorboard\n",
            "    Found existing installation: tensorboard 2.9.1\n",
            "    Uninstalling tensorboard-2.9.1:\n",
            "      Successfully uninstalled tensorboard-2.9.1\n",
            "  Attempting uninstall: keras\n",
            "    Found existing installation: keras 2.9.0\n",
            "    Uninstalling keras-2.9.0:\n",
            "      Successfully uninstalled keras-2.9.0\n",
            "  Attempting uninstall: tensorflow\n",
            "    Found existing installation: tensorflow 2.9.2\n",
            "    Uninstalling tensorflow-2.9.2:\n",
            "      Successfully uninstalled tensorflow-2.9.2\n",
            "Successfully installed keras-2.8.0 tensorboard-2.8.0 tensorflow-2.8.4 tensorflow-estimator-2.8.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tensorflow-text==2.8.*\n",
            "  Downloading tensorflow_text-2.8.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (4.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.9 MB 39.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow<2.9,>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text==2.8.*) (2.8.4)\n",
            "Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text==2.8.*) (0.12.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (57.4.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.50.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.1.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.27.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.6.3)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (4.1.1)\n",
            "Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.12)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.21.6)\n",
            "Requirement already satisfied: keras<2.9,>=2.8.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.8.0)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.14.1)\n",
            "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.3.0)\n",
            "Requirement already satisfied: tensorboard<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.8.0)\n",
            "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (14.0.6)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.1.2)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.1.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.2.0)\n",
            "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.19.6)\n",
            "Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.4.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.3.0)\n",
            "Requirement already satisfied: tensorflow-estimator<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.8.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.38.4)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.5.2)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.4.6)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.8.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.4.1)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.6.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.14.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (5.2.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (4.13.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.10.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (2022.9.24)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.*) (3.2.2)\n",
            "Installing collected packages: tensorflow-text\n",
            "Successfully installed tensorflow-text-2.8.2\n"
          ]
        }
      ],
      "source": [
        "!pip install -U \"tensorflow==2.8.*\"\n",
        "!pip install -U \"tensorflow-text==2.8.*\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text"
      ],
      "metadata": {
        "id": "TiTFW5nWfuIO"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_x_train = pd.read_csv(\"X_train.csv\")\n",
        "print(df_x_train)\n",
        "df_y_train = pd.read_csv(\"y_train.csv\")\n",
        "print(df_y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4rVScLX6gP6J",
        "outputId": "0a513433-f9b8-499d-b17c-51352cf5203a"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "        id              keyword            location  \\\n",
            "0     2714              crushed         Chicago, IL   \n",
            "1     7361           obliterate         Cymru araul   \n",
            "2     1290               bloody             Chicago   \n",
            "3     1932  burning%20buildings                 NaN   \n",
            "4     4041             disaster         Calgary, AB   \n",
            "...    ...                  ...                 ...   \n",
            "6085  7532          oil%20spill    Street of Dallas   \n",
            "6086  1821               burned  toronto Û¢ dallas   \n",
            "6087  9504            terrorist                 NaN   \n",
            "6088  2797               curfew            IM LOST    \n",
            "6089  3667              destroy                 NaN   \n",
            "\n",
            "                                           cleaned_text  \n",
            "0       jimmyfallon crush squirrel bone mortar pestl...  \n",
            "1      mccainenl think spectacular look stonewal rio...  \n",
            "2     can t bloodi wait   soni set date stephen king...  \n",
            "3     protest ralli stone mountain  atleast they r b...  \n",
            "4      rbcinsur quot websit   disaster  tri 3 browse...  \n",
            "...                                                 ...  \n",
            "6085     refugio oil spill may costlier bigger project   \n",
            "6086  hate im tri straighten hair brother come swoop...  \n",
            "6087               pull fight hoe call zayn terrorist    \n",
            "6088             da judg gave di girl 5pm curfew         \n",
            "6089  get   150 free bet regist boylesport casino lu...  \n",
            "\n",
            "[6090 rows x 4 columns]\n",
            "        id  target\n",
            "0     2714       0\n",
            "1     7361       1\n",
            "2     1290       0\n",
            "3     1932       0\n",
            "4     4041       0\n",
            "...    ...     ...\n",
            "6085  7532       1\n",
            "6086  1821       0\n",
            "6087  9504       1\n",
            "6088  2797       0\n",
            "6089  3667       0\n",
            "\n",
            "[6090 rows x 2 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_column = df_x_train[\"cleaned_text\"]\n",
        "y_column = df_y_train[\"target\"]\n",
        "preprocess_url = \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\"\n",
        "encoder_url = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4\"\n",
        "bert_preprocess = hub.KerasLayer(preprocess_url)\n",
        "bert_encoder = hub.KerasLayer(encoder_url)"
      ],
      "metadata": {
        "id": "NtInb1B4geC7"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf"
      ],
      "metadata": {
        "id": "qxSd_jxWgh00"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='dataset_input')\n",
        "preprocessed_text = bert_preprocess(text_input)\n",
        "outputs = bert_encoder(preprocessed_text)\n",
        "\n",
        "l = tf.keras.layers.Dropout(0.1, name=\"dropout1\")(outputs['pooled_output'])\n",
        "l = tf.keras.layers.Dense(100,  activation='relu', name='output1')(l)\n",
        "l = tf.keras.layers.Dropout(0.2, name=\"dropout2\")(l)\n",
        "l = tf.keras.layers.Dense(50, activation='relu', name='output2')(l)\n",
        "l = tf.keras.layers.Dropout(0.2, name=\"dropout3\")(l)\n",
        "l = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(l)\n",
        "\n",
        "model = tf.keras.Model(inputs=[text_input], outputs = [l])"
      ],
      "metadata": {
        "id": "1KlaN6sfhIpM"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dpPv6D2chP-3",
        "outputId": "c1dffbc5-295f-4991-ebe7-69830762a43e"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " dataset_input (InputLayer)     [(None,)]            0           []                               \n",
            "                                                                                                  \n",
            " keras_layer (KerasLayer)       {'input_type_ids':   0           ['dataset_input[0][0]']          \n",
            "                                (None, 128),                                                      \n",
            "                                 'input_mask': (Non                                               \n",
            "                                e, 128),                                                          \n",
            "                                 'input_word_ids':                                                \n",
            "                                (None, 128)}                                                      \n",
            "                                                                                                  \n",
            " keras_layer_1 (KerasLayer)     {'encoder_outputs':  109482241   ['keras_layer[0][0]',            \n",
            "                                 [(None, 128, 768),               'keras_layer[0][1]',            \n",
            "                                 (None, 128, 768),                'keras_layer[0][2]']            \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768)],                                               \n",
            "                                 'sequence_output':                                               \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 'pooled_output': (                                               \n",
            "                                None, 768),                                                       \n",
            "                                 'default': (None,                                                \n",
            "                                768)}                                                             \n",
            "                                                                                                  \n",
            " dropout1 (Dropout)             (None, 768)          0           ['keras_layer_1[0][13]']         \n",
            "                                                                                                  \n",
            " output1 (Dense)                (None, 100)          76900       ['dropout1[0][0]']               \n",
            "                                                                                                  \n",
            " dropout2 (Dropout)             (None, 100)          0           ['output1[0][0]']                \n",
            "                                                                                                  \n",
            " output2 (Dense)                (None, 50)           5050        ['dropout2[0][0]']               \n",
            "                                                                                                  \n",
            " dropout3 (Dropout)             (None, 50)           0           ['output2[0][0]']                \n",
            "                                                                                                  \n",
            " output (Dense)                 (None, 1)            51          ['dropout3[0][0]']               \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,564,242\n",
            "Trainable params: 82,001\n",
            "Non-trainable params: 109,482,241\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from keras import backend as K\n",
        "\n",
        "def my_recall(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def my_precision(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "def my_f1(y_true, y_pred):\n",
        "    precision = my_precision(y_true, y_pred)\n",
        "    recall = my_recall(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
      ],
      "metadata": {
        "id": "Wh0uUSE3TLGu"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='Adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy',my_recall,my_precision,my_f1])\n",
        "result = model.fit(x_column, y_column, epochs=10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nS8Kv9l4hS5C",
        "outputId": "e9d4a54a-9d0c-4299-b469-2f69680db55d"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "191/191 [==============================] - 71s 356ms/step - loss: 0.5426 - accuracy: 0.7386 - my_recall: 0.5989 - my_precision: 0.7536 - my_f1: 0.6498\n",
            "Epoch 2/10\n",
            "191/191 [==============================] - 65s 343ms/step - loss: 0.5471 - accuracy: 0.7373 - my_recall: 0.5813 - my_precision: 0.7636 - my_f1: 0.6443\n",
            "Epoch 3/10\n",
            "191/191 [==============================] - 65s 340ms/step - loss: 0.5386 - accuracy: 0.7420 - my_recall: 0.5997 - my_precision: 0.7664 - my_f1: 0.6566\n",
            "Epoch 4/10\n",
            "191/191 [==============================] - 65s 342ms/step - loss: 0.5358 - accuracy: 0.7424 - my_recall: 0.5872 - my_precision: 0.7763 - my_f1: 0.6534\n",
            "Epoch 5/10\n",
            "191/191 [==============================] - 65s 340ms/step - loss: 0.5440 - accuracy: 0.7356 - my_recall: 0.5888 - my_precision: 0.7547 - my_f1: 0.6430\n",
            "Epoch 6/10\n",
            "191/191 [==============================] - 66s 345ms/step - loss: 0.5387 - accuracy: 0.7415 - my_recall: 0.5983 - my_precision: 0.7620 - my_f1: 0.6520\n",
            "Epoch 7/10\n",
            "191/191 [==============================] - 65s 340ms/step - loss: 0.5327 - accuracy: 0.7458 - my_recall: 0.6056 - my_precision: 0.7568 - my_f1: 0.6601\n",
            "Epoch 8/10\n",
            "191/191 [==============================] - 65s 342ms/step - loss: 0.5261 - accuracy: 0.7516 - my_recall: 0.6168 - my_precision: 0.7683 - my_f1: 0.6714\n",
            "Epoch 9/10\n",
            "191/191 [==============================] - 65s 342ms/step - loss: 0.5268 - accuracy: 0.7512 - my_recall: 0.6152 - my_precision: 0.7740 - my_f1: 0.6739\n",
            "Epoch 10/10\n",
            "191/191 [==============================] - 65s 341ms/step - loss: 0.5264 - accuracy: 0.7496 - my_recall: 0.6119 - my_precision: 0.7674 - my_f1: 0.6670\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "nurUvsI9THen"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_x_val = pd.read_csv(\"X_val.csv\")\n",
        "print(df_x_val)\n",
        "df_y_val = pd.read_csv(\"y_val.csv\")\n",
        "print(df_y_val)"
      ],
      "metadata": {
        "id": "DLFCkoP5458D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "24c5cad1-7cbd-45ca-caa8-ba7e675614b0"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "        id         keyword                       location  \\\n",
            "0     6959        massacre                       Colorado   \n",
            "1     6453         injured                      Worldwide   \n",
            "2     5764  forest%20fires                            NaN   \n",
            "3     3841        detonate                  Morioh, Japan   \n",
            "4     2341        collapse                         Europe   \n",
            "...    ...             ...                            ...   \n",
            "1518  7581        outbreak                    FCT, Abuja    \n",
            "1519  4790       evacuated                     Manchester   \n",
            "1520  1382      body%20bag  sitting on Eddie Vedders lap,   \n",
            "1521  4068       displaced       Pedophile hunting ground   \n",
            "1522  1105       blew%20up                            NaN   \n",
            "\n",
            "                                           cleaned_text  \n",
            "0     colorado movi massacr trial juror reach verdic...  \n",
            "1     top stori   googl 4 dead dozen injur gaza blas...  \n",
            "2     group florida forest servic firefight could de...  \n",
            "3       tinyjecht anoth stand user  deton killer queen   \n",
            "4     greece  tax revenu collaps debt crisi continu ...  \n",
            "...                                                 ...  \n",
            "1518  8th person die ny legionnaires  diseas outbrea...  \n",
            "1519  trafford centr film fan angri odeon cinema eva...  \n",
            "1520   questergirl ditto had  way feel drank vodka i...  \n",
            "1521   potu strategicpati strategi genocide  refugee...  \n",
            "1522  queen gambit went well anakin blew droid contr...  \n",
            "\n",
            "[1523 rows x 4 columns]\n",
            "        id  target\n",
            "0     6959       1\n",
            "1     6453       1\n",
            "2     5764       1\n",
            "3     3841       0\n",
            "4     2341       1\n",
            "...    ...     ...\n",
            "1518  7581       1\n",
            "1519  4790       1\n",
            "1520  1382       0\n",
            "1521  4068       1\n",
            "1522  1105       0\n",
            "\n",
            "[1523 rows x 2 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_val = df_x_val[\"cleaned_text\"]\n",
        "print(x_val)\n",
        "y_val = df_y_val[\"target\"]\n",
        "print(y_val)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nUqBowa2XIt4",
        "outputId": "e6a98237-f2c4-4b6b-9bf7-fb0a5d088eea"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0       colorado movi massacr trial juror reach verdic...\n",
            "1       top stori   googl 4 dead dozen injur gaza blas...\n",
            "2       group florida forest servic firefight could de...\n",
            "3         tinyjecht anoth stand user  deton killer queen \n",
            "4       greece  tax revenu collaps debt crisi continu ...\n",
            "                              ...                        \n",
            "1518    8th person die ny legionnaires  diseas outbrea...\n",
            "1519    trafford centr film fan angri odeon cinema eva...\n",
            "1520     questergirl ditto had  way feel drank vodka i...\n",
            "1521     potu strategicpati strategi genocide  refugee...\n",
            "1522    queen gambit went well anakin blew droid contr...\n",
            "Name: cleaned_text, Length: 1523, dtype: object\n",
            "0       1\n",
            "1       1\n",
            "2       1\n",
            "3       0\n",
            "4       1\n",
            "       ..\n",
            "1518    1\n",
            "1519    1\n",
            "1520    0\n",
            "1521    1\n",
            "1522    0\n",
            "Name: target, Length: 1523, dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "result=model.evaluate(x_val,y_val)\n",
        "print(result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TPXWGQI7Xe9f",
        "outputId": "12e2a090-e3ce-4c4e-dab3-5b6bed6f90aa"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "48/48 [==============================] - 17s 334ms/step - loss: 0.5169 - accuracy: 0.7525 - my_recall: 0.5842 - my_precision: 0.7729 - my_f1: 0.6546\n",
            "[0.5168598890304565, 0.7524622678756714, 0.5841571092605591, 0.7728593945503235, 0.6546135544776917]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "OP4WVoSao-Ji"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}