BDFoodAppSentiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Administrator\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\Administrator\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package maxent_ne_chunker to\n",
      "[nltk_data]     C:\\Users\\Administrator\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package maxent_ne_chunker is already up-to-date!\n",
      "[nltk_data] Downloading package words to\n",
      "[nltk_data]     C:\\Users\\Administrator\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n",
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     C:\\Users\\Administrator\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.style.use('ggplot')\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('maxent_ne_chunker')\n",
    "nltk.download('words')\n",
    "nltk.download('vader_lexicon')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in c:\\programdata\\anaconda3\\lib\\site-packages (4.26.1)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (1.21.5)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (2.28.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (2022.7.9)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (4.64.1)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (0.12.0)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (0.13.2)\n",
      "Requirement already satisfied: filelock in c:\\programdata\\anaconda3\\lib\\site-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\programdata\\anaconda3\\lib\\site-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.3.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
      "Requirement already satisfied: colorama in c:\\programdata\\anaconda3\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.5)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->transformers) (1.26.11)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->transformers) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->transformers) (2022.9.14)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "from scipy.special import softmax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Dataset and PreProcessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15017, 5)\n"
     ]
    }
   ],
   "source": [
    "url = 'https://github.com/errhythm/BDFoodAppSentiment/raw/main/Dataset.csv'\n",
    "df = pd.read_csv(url)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>createdAt</th>\n",
       "      <th>text</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>ratingvalue</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-02-04T11:10:14.474Z</td>\n",
       "      <td>he received my order around 2.30 pm but delive...</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-07-16T05:59:24Z</td>\n",
       "      <td>taste was not that good. and the amount of cof...</td>\n",
       "      <td>Israel</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-04-24T12:15:02.221Z</td>\n",
       "      <td>It is better if the girl is a little bigger</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-11-16T15:55:42.189Z</td>\n",
       "      <td>take too much time. today i feel very angry</td>\n",
       "      <td>Zerin Ibnat</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-06-30T05:53:21Z</td>\n",
       "      <td>Coleslaw was not fresh and it had a bad smell....</td>\n",
       "      <td>faria</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  createdAt  \\\n",
       "0  2022-02-04T11:10:14.474Z   \n",
       "1      2022-07-16T05:59:24Z   \n",
       "2  2022-04-24T12:15:02.221Z   \n",
       "3  2021-11-16T15:55:42.189Z   \n",
       "4      2022-06-30T05:53:21Z   \n",
       "\n",
       "                                                text reviewerName  \\\n",
       "0  he received my order around 2.30 pm but delive...    Anonymous   \n",
       "1  taste was not that good. and the amount of cof...       Israel   \n",
       "2        It is better if the girl is a little bigger    Anonymous   \n",
       "3        take too much time. today i feel very angry  Zerin Ibnat   \n",
       "4  Coleslaw was not fresh and it had a bad smell....        faria   \n",
       "\n",
       "   ratingvalue  id  \n",
       "0            4   0  \n",
       "1            1   1  \n",
       "2            5   2  \n",
       "3            1   3  \n",
       "4            1   4  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sample(frac=1).reset_index(drop=True)\n",
    "df = df.drop(['Unnamed: 0'], axis=1)\n",
    "df['id'] = df.index\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = df[\"ratingvalue\"].value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0gAAAHTCAYAAAAd2ThyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABE2UlEQVR4nO3deVwW5f7/8fcN3IIsAooKCCgumIqISprLUbNsMb8q6bHFSkP96cnUR5t2onI5mlrnZGqdsnCLyuOSpKWZ5VJq5Zrp0dwl9xQVkUC5gfn94TCnWxBBEQxfz8ejR94z18x85r4v9H5zXTNjMwzDEAAAAABALmVdAAAAAADcLAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISACAAq1YsUJt27aVv7+/bDabunfvXtYlFVtycrJsNpv69u1b1qVct/J0LgBwMyMgAbgl7Nq1S0OGDFFkZKR8fX1VoUIFBQcH64EHHtD06dN14cKFsi7xqmbNmiWbzaZZs2bd8GP9+uuv+r//+z/t379fTz75pEaOHKmHH3640G1Wr14tm83m9J/dbleNGjXUo0cPrVu37obXjRsnJydHH3zwgdq3b6/KlSvLbrerWrVqioqKUv/+/bV48WKn9qXZXwGgJLmVdQEAcKONGTNGo0ePVm5uru644w716dNHPj4++u233/Tdd9+pf//+evfdd7Vp06ayLvWm8fXXXyszM1MJCQl69NFHi7VtzZo1rVGOjIwMbd68WQsXLtRnn32mefPmqUePHjeg4oLVqFFDv/zyi3x9fUvtmOVRTk6OunTpomXLlsnPz08PPPCAQkJCdObMGe3bt0+JiYnatWuXunbtWtalAsB1IyABKNfGjRunkSNHKjQ0VPPnz1fLli3ztVm2bJlef/31Mqju5nXs2DFJUnBwcLG3rVWrlkaNGuW0bMKECfr73/+uF154oVQDkt1u12233VZqxyuv5syZo2XLlqlJkyb69ttv8wXOs2fPavPmzWVUHQCULKbYASi3kpOTNXr0aNntdi1durTAcCRJ9913n7788st8y+fOnau//OUv8vX1VcWKFRUZGanXXnutwOl4NptNHTp0KHD/ffv2lc1mU3JyslNtedeTJCcn6+GHH1ZAQIA8PDzUvHnzfNOVOnTooCeffFKS9OSTTzpNY/vjfgtTlPPJmyY3cuRISdKdd95pHWf16tVFOk5B+vfvL0k6ePCgUlJS8q2fM2eO7rzzTvn7+8vDw0MNGjTQ2LFjdfHiRavN0aNH5erqqmbNml3xOHfffbdsNpv++9//Sir8up2MjAyNHz9e0dHR8vLykre3t1q1aqU5c+Y4tdu9e7dsNpsee+wxp+V79uyx3ps1a9Y4rRs+fLhsNptWrlxpLfvpp5/00EMPqWbNmnJ3d1eVKlUUFRWlYcOGyeFwXPGcCrJr1y51795dlStXlpeXl9q2bavly5c7tXnvvfdks9k0ZsyYAvdx4sQJ2e12NW7c+KrHy5se2bdv3wJH4/z9/XX33Xdbr4vSX48dO6YxY8aoTZs2CgwMtKa9PvLII9qxY0e+Y/zxs9y1a5d69uypqlWrysXFxeqb+/btU//+/VWnTh15eHjI399fDRo00MCBA3X69OmrnicASIwgASjHZs6cKYfDoYcffliRkZGFtnV3d3d6PWLECL3++uuqWrWqevfuLS8vLy1dulTx8fFatmyZvvnmG1WoUOG6a/z111/VokUL1a5dW48//rjOnDmjuXPnqnv37vr666911113Sbr0xdTPz0+LFi1St27dFB0dbe3Dz8/vqscp6vnUqlVLI0eO1OrVq/Xtt9+qT58+qlWrliRZ/78Wubm51p/d3Jz/6enXr59mzJih0NBQ9ejRQ76+vvrxxx/1yiuvaMWKFVq+fLl1LdPdd9+t5cuXa/v27fm+2B85ckSrVq1S8+bNr/p5p6amqmPHjvrpp5/UvHlzxcXFKTc3V1999ZUeffRR7dixQ2PHjpUk1a9fXzVq1HAKO5KcXq9YsUJ/+ctfnNZ5eHiodevWkqStW7eqVatWcnFxUdeuXRUeHq60tDTt27dP7777rsaNGye73V6k9/LgwYNq1aqVIiMjNXDgQB0/flxz587V/fffr08++UQPPfSQJOmxxx7TiBEjlJCQoPj4eLm6ujrtZ/r06crOztbAgQOvesyqVatKuhQKi6Io/fW7777ThAkTdOedd6pHjx7y8vLS3r17tWDBAi1evFjr1q1z2i7Pvn37dMcdd6h+/fp67LHHlJ6eLh8fHx07dkwtWrTQ+fPn1blzZ/Xs2VMXLlzQwYMH9dFHH2nIkCGqUqVKkeoHcIszAKCcuvPOOw1JxgcffFCs7dauXWtIMmrWrGn89ttv1nKHw2F07tzZkGSMHTvWaRtJRvv27QvcX58+fQxJxsGDB61lBw8eNCQZkoxRo0Y5tV+2bJkhybjvvvucls+cOdOQZMycOfOGn8/IkSMNScaqVauKfJxVq1Zd8X147bXXDElGo0aNnJbnnVPPnj2NzMzMAmuYNGmStezjjz82JBnPPfdcvmOMHz/ekGRMmTLFWpb3Pvfp08epbd5n8s9//tNpeWZmpnHvvfcaNpvN2LJli7X88ccfNyQZO3bssJb17NnTqF69utG4cWOjbdu21vIzZ84YLi4uRseOHa1lzzzzjCHJSEpKylf3mTNnjJycnHzLL/fHPvP88887rdu4caPh5uZm+Pn5GefOnbOWDx482JBkfP75507tc3JyjFq1ahmenp5GamrqVY+9detWw263Gzabzejdu7cxb94848CBA4Vuc7X++ttvvxlpaWn5lm/evNnw9PQ07r33Xqflfzz/v//97/m2mzx5cr7+kic9Pd3IyMgotF4AyMMUOwDl1okTJyRJISEhxdpu5syZkqSXX35Z1apVs5a7ubnpzTfflIuLi6ZPn14iNdaqVUsvv/yy07J7771XYWFh2rhxY4kcozTPR7o0FWrUqFEaNWqUhg8frrvuuksvvfSSfHx89N577zm1nTx5sux2uz744AN5eHg4rXvllVdUpUoVffzxx9ay2NhYVapUSR9//LFycnKc2n/44Yey2+165JFHCq3v9OnT+uijj3T77bfrueeec1rn4eGhiRMnyjAMffLJJ9byvJG8FStWSJIMw9CqVat011136e6779b69euVnp4uSVq1apVyc3OtbaRLUzAlydPTM189/v7+cnEp+j/Hvr6+evXVV52WxcTEqHfv3kpNTVVSUpK1/KmnnpIkvf/++07tv/rqK2tqZ1FuYNGkSRN98sknCgwM1Mcff6xevXqpdu3aCggIUI8ePbR06dIi15+nWrVq8vHxybe8WbNm6tixo1avXl3g1MPq1atbU0D/qLD32MvLSxUrVix2jQBuTUyxA1BuGYYh6X9fnIrqp59+knTp+pvL1a9fXyEhITp48KBSU1OLNL2tMNHR0fmmPklSaGiofvjhh+vad57SPB/p0rTB0aNHOy3z9/fXypUrnaZMZWRk6Oeff1ZAQIDeeuutAvfl7u6uXbt2Wa8rVqyoXr16KSEhQV999ZU6d+4sSdq4caN++eUXxcbGKiAgoND6Nm7caIWry28mIcn6Uv7H4+aFnZUrV2rIkCH6+eefdfr0ad11112qXr26Jk2apDVr1uj++++3pt517NjR2v7hhx/W5MmT1b17d/31r3/VXXfdpTZt2qhOnTqF1lqQZs2aFRgsOnTooNmzZ+unn35Snz59JEkNGzZU+/bttXTpUh05csT6ZcG0adMkqUjT6/L07NlT3bp106pVq7R27Vr99NNPWrt2rRYuXKiFCxcqLi5OCQkJxfp5W7Jkid577z1t2rRJKSkpys7OdlqfkpKioKAgp2VNmjTJNyVWkrp27aqXXnpJgwcP1tdff61OnTqpTZs2atiwYbH/DgBwayMgASi3goODtWvXLh05cqRY2507d06SFBgYWOD6oKAgHTp0SOfOnbvuQHGl3967ubk5XbdzPUrzfCSpffv21kXzZ86c0fz58zV06FB169ZNGzdutEaxzp49K8MwdOrUqXyBqjB9+vRRQkKCZs+ebQWk2bNnW+uuJu9i/Y0bNxY6Spc3IiRdGoWsV6+eVq9erZycHGskqWPHjqpSpYrc3Ny0YsUK3X///VqxYoUqVaqkmJgYa/vbb79da9as0bhx4zR//nx9+OGHkqTbbrtNo0aNsq4bKorq1asXuDzv8837vPP87W9/07fffqvp06dr5MiROnbsmL744gtFR0erRYsWRT6udOmugPfcc4/uueceSZdu//3pp58qLi5OM2bMUNeuXdWtW7ci7WvKlCkaNmyY/P391alTJ4WFhcnT01M2m02fffaZfv75Z6ebdFx+nperWbOmNmzYoFGjRmnZsmVasGCBpEu/bBg+fLiefvrpYp0rgFsXU+wAlFtt27aV9L9pUUWVF1rypuhd7vjx407tpEujVJf/9jtPampqsY5f0q7lfEpK5cqVNXDgQL355ps6dOiQ/va3v+Wrq2nTpjIMo9D//qht27aqU6eOFi1apNTUVDkcDv3nP/9RQECAFZgKk3fcZ555ptBjrlq1ymm7jh07KjU1VVu2bNGKFStUp04d1apVSz4+PmrRooW++eYbHT9+XLt27VK7du3y3YyiVatW+uKLL3T27FmtW7dOr7zyik6cOKFHHnkk3w0gCvPbb78VuDzv8738c3zwwQcVGBiohIQE5eTkWP8fNGhQkY95Ja6ururVq5eeeeYZSUX/WcvOztbIkSMVGBioHTt2aO7cuXrjjTc0evRojRo16oohUCp8RLhBgwaaO3euTp8+rU2bNmnChAnKzc3VkCFDrKmmAHA1BCQA5daTTz4pu92uTz/9VDt37iy07R9/U920aVNJKvC21vv27dORI0cUHh7uNNri7++vw4cP52ufk5OjrVu3XlP9l8ubinf5tTdXcy3nU9IGDRqkRo0aaeHChdYto729vdWoUSPt2LFDZ86cKdb+nnjiCV28eFFz587VF198odOnT+vRRx8t0p3gWrRoIRcXl3y35r6avGl2X331ldasWeN0jdFdd92lbdu26T//+Y9T24K4u7urdevWGjNmjKZMmSLDMPTZZ58VuY4tW7bo/Pnz+Zbnfb55n3ceu92u/v3768iRI/riiy+UkJAgb2/vYj8AuDB5U/7+GGYL668pKSlKTU1V69at802hS09P15YtW66rHjc3NzVv3lwjRoywbtv+x2uzAKAwBCQA5VbeA0uzsrL0wAMPaNOmTQW2W7Zsme6//37rdVxcnCRp7NixOnXqlLU8JydHzz//vHJzc9WvXz+nfbRs2VKHDh3K9yyasWPH6tdffy2R88m7RXFBQaww13I+Jc3V1dWaRvfSSy9Zy5999lllZWUpLi6uwJG2s2fPFvhluU+fPrLZbPrwww+t6WoFPeuoINWqVVPv3r21adMm/eMf/yhw5G///v06ePCg07K8Z0JNnTpV6enp+QKSYRiaMGGCJOfrjyRpzZo1+aa+Sf8bDbr8BhWFOXfuXL5nG23atEkff/yxfH19FRsbm2+bgQMHytXVVX/72990+PBh9e7du8DrmK5kzpw5+vrrrwuc9nnixAl98MEHkqR27dpZywvrr9WqVZOnp6c2bdrkNJXR4XBo2LBhBT4r62o2bNhQ4OjatbzHAG5tXIMEoFx76aWXlJ2drdGjR+v2229X69atFRMTI29vb/3222/67rvvtHfvXqfrRVq3bq3hw4fr9ddfV2RkpHr27CkvLy99+eWX+u9//6u2bdvqhRdecDrO888/r6+++krdunXTQw89pMqVK+v777/XwYMH1aFDh+t6yGqeVq1aydPTU2+99ZZOnz5tTUMaMmRIodPjruV8boQHH3xQ0dHR+u677/TVV1/p3nvvVVxcnDZv3qx///vfqlOnjnUHvzNnzujgwYP67rvv9OSTT+a7+13NmjWta53c3NzUuHHjfCMnhXn77be1d+9evfrqq0pMTFTbtm1VvXp1HTt2TL/88os2btyoOXPmKDw83NomICBAUVFR+vnnn2Wz2ZxuepH32Zw8eVJVq1bN94ymf/3rX1q+fLk6dOig2rVry9vbWzt27NCXX34pPz8//b//9/+KXHu7du2UkJCg9evXq02bNtZzkHJzczVt2jRVqlQp3zYhISHq0qWLFi1aJKl4N2eQpPXr12vy5MkKDAxU27Ztrffl4MGDWrJkiTIzM9WtWzf17Nkz33typf46dOhQTZgwQY0bN1a3bt2UlZWlVatW6cyZM7rzzjvzTXG8mk8++UTvvPOO2rdvr7p168rf31/79+/X559/Lnd3dw0bNqxY+wNwCyvNe4oDQFnZuXOn8fTTTxuNGjUyfHx8DLvdbgQGBhr33XefkZCQYFy4cCHfNnPmzDHatGljeHt7G+7u7kbDhg2NsWPH5nteT57FixcbzZs3N9zd3Y3KlSsbDz30kJGcnFzoc5Aufz5Pnvbt2xsF/RX95ZdfGnfccYfh5eVlPRPmj/stTHHOp6Sfg5Rn8eLFhiQjJibGafnnn39uPPDAA0bVqlUNu91uVK9e3bj99tuN+Ph445dffilwX3nP2VEBzzPKU9j7fPHiRWPq1KlGq1atjEqVKhkVKlQwQkNDjY4dOxqTJk0yUlJS8m3z7LPPGpKMqKiofOvuueceQ5LRq1evfOu++uoro2/fvkaDBg2MSpUqGZ6enkZERIQxZMgQIzk5ucDaCzuXX375xejatavh5+dnVKxY0WjdurWxbNmyQrdftGiRIclo0aJFkY73R4cOHTLefvtto3v37kZERITTz9D9999vJCYmFvgsp8L6q8PhMP71r38ZDRo0MDw8PIzq1asbjz322DX/zPz444/GoEGDjKioKMPf39/w8PAw6tSpY/Tt29fYvn17sc8ZwK3LZhiXXf0KAADKnVdffVX/+Mc/NH36dGvaJQAgPwISAADlXFpamurWravc3FwdPnyYh6YCQCG4BgkAgHLq888/16ZNm/TFF1/o1KlTmjRpEuEIAK6CgAQAQDn16aefavbs2QoMDNTLL7+soUOHlnVJAHDTY4odAAAAAJh4DhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAAKZyfxe7s2fPKjs7u6zL+FOpWrWqTp06VdZl4BZAX0Npoa+htNDXUFroa8Xj5uYmf3//orW9wbWUuezsbDkcjrIu40/DZrNJuvS+cYND3Ej0NZQW+hpKC30NpYW+dmMxxQ4AAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADA5FbWBdyqcgZ0LesSruhwWRdwBa4fLC7rEgAAAFDOMYIEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACa34jSeN2+eFixY4LTM19dXH3zwgSTJMAzNnz9fK1asUHp6uurVq6d+/fopNDTUau9wOJSYmKh169YpKytLkZGR6t+/v6pUqWK1SU9P18yZM7Vp0yZJUkxMjOLi4uTl5XXNJwoAAAAAV1OsgCRJoaGheuWVV6zXLi7/G4RatGiRlixZoqeeekpBQUFauHChxo4dq7feeksVK1aUJM2aNUubN2/WsGHD5OPjow8//FATJkzQxIkTrX1NmTJFp0+fVnx8vCRp2rRpmjp1ql588cXrOlkAAAAAKEyxp9i5uLjIz8/P+q9SpUqSLo0eLV26VLGxsWrZsqXCwsI0ePBgXbx4UWvXrpUkZWRkaOXKlXriiScUFRWl8PBwDRkyRIcOHdK2bdskSUeOHNHWrVs1aNAgRUREKCIiQgMHDtSWLVt07NixEjx1AAAAAHBW7BGkEydOaODAgXJzc1O9evX0yCOPqHr16jp58qRSU1PVpEkTq63dblfDhg21e/duderUSQcOHFBOTo6ioqKsNpUrV1ZYWJj27Nmj6Oho7dmzR56enqpXr57VJiIiQp6entq9e7eCg4MLrMvhcMjhcFivbTabNWpls9mKe5q4CfE5li95nyefK240+hpKC30NpYW+dmMVKyDVq1dPgwcPVnBwsFJTU7Vw4UK9/PLLevPNN5Wamirp0jVJf+Tr66uUlBRJUmpqqtzc3OTt7Z2vTd72qamp+fZxeZuCJCUlOV0fFR4erokTJ6pq1arFOcVSc7isC/gTCgoKKusScAMEBgaWdQm4RdDXUFroaygt9LUbo1gBqWnTptafw8LCFBERoSFDhujbb7+1RnwuT7KGYVx1v0VtU1hKjo2NVZcuXazXeW1PnTql7Ozsq+4fN7/jx4+XdQkoQTabTYGBgTpx4kSR/g4ArhV9DaWFvobSQl8rPjc3tyIPnBR7it0feXh4KCwsTMePH9ftt98u6dIIkL+/v9UmLS3NGhHy8/NTdna20tPTnUaR0tLSVL9+favNuXPn8h3rj/spiN1ul91uL3AdHad84HMsnwzD4LNFqaCvobTQ11Ba6Gs3xnU9B8nhcOjo0aPy9/dXtWrV5OfnZ91sQZKys7O1c+dOK/zUrl1brq6uTm3Onj2rQ4cOKSIiQtKl640yMjK0b98+q83evXuVkZFh7QcAAAAAboRijSB9+OGHiomJUUBAgM6dO6dPP/1UmZmZat++vWw2mzp37qykpCQFBQUpMDBQSUlJcnd3V9u2bSVJnp6e6tixoxITE+Xj4yNvb28lJiYqLCzMunFDSEiIoqOjNW3aNA0YMECS9P7776tZs2ZXvEEDAAAAAJSEYgWkM2fOaPLkyUpLS1OlSpVUr149jRs3zprP161bN2VlZSkhIUG///676tatq/j4eOtucpLUp08fubq6atKkSdaDYkeMGOH0PKWhQ4dqxowZGjdunCSpefPm6tevX0mcLwAAAABckc0o5xMXT5065XT775tFzoCuZV3Cn47rB4vLugSUIJvNpqCgIB0/fpz507ih6GsoLfQ1lBb6WvHZ7fYi36Thuq5BAgAAAIDyhIAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmNzKugAAAADgZpQzoGtZl3BFh8u6gCtw/WBxWZdw3RhBAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMB0XQ+KTUpK0pw5c9S5c2f17dtXkmQYhubPn68VK1YoPT1d9erVU79+/RQaGmpt53A4lJiYqHXr1ikrK0uRkZHq37+/qlSpYrVJT0/XzJkztWnTJklSTEyM4uLi5OXldT0lAwAAAMAVXfMI0r59+/TNN9+oZs2aTssXLVqkJUuWKC4uTuPHj5efn5/Gjh2rzMxMq82sWbO0YcMGDRs2TGPGjNGFCxc0YcIE5ebmWm2mTJmi5ORkxcfHKz4+XsnJyZo6deq1lgsAAAAAV3VNAenChQuaOnWqBg4c6DSiYxiGli5dqtjYWLVs2VJhYWEaPHiwLl68qLVr10qSMjIytHLlSj3xxBOKiopSeHi4hgwZokOHDmnbtm2SpCNHjmjr1q0aNGiQIiIiFBERoYEDB2rLli06duxYCZw2AAAAAOR3TVPsEhIS1LRpU0VFRWnhwoXW8pMnTyo1NVVNmjSxltntdjVs2FC7d+9Wp06ddODAAeXk5CgqKspqU7lyZYWFhWnPnj2Kjo7Wnj175OnpqXr16lltIiIi5Onpqd27dys4ODhfTQ6HQw6Hw3pts9lUsWJF68/48+NzLF/yPk8+V9xo9DWUFvoaUD76f7ED0rp163Tw4EGNHz8+37rU1FRJkq+vr9NyX19fpaSkWG3c3Nzk7e2dr03e9qmpqfn2cXmbyyUlJWnBggXW6/DwcE2cOFFVq1Yt6qmVqsNlXcCfUFBQUFmXgBsgMDCwrEvALYK+htJCXys/+L5WfOXh+1qxAlJKSopmzZql+Ph4VahQ4YrtLk+OhmFcdd9FbXOlVBobG6suXbrkq+HUqVPKzs6+6r5x8zt+/HhZl4ASZLPZFBgYqBMnThTp5x+4VvQ1lBb6GnDzfl9zc3Mr8sBJsQLSgQMHdO7cOb344ovWstzcXP3yyy9atmyZ3nrrLUmXRoD8/f2tNmlpadaIkJ+fn7Kzs5Wenu40ipSWlqb69etbbc6dO5fv+H/cz+XsdrvsdnuB6/hLqnzgcyyfDMPgs0WpoK+htNDXcCsrD32/WAGpcePG+uc//+m07N1331VwcLC6deum6tWry8/PT9u2bVN4eLgkKTs7Wzt37lTv3r0lSbVr15arq6u2bdum1q1bS5LOnj2rQ4cOWW0iIiKUkZGhffv2qW7dupKkvXv3KiMjwwpRAAAAAFDSihWQKlasqLCwMKdl7u7u8vHxsZZ37txZSUlJCgoKUmBgoJKSkuTu7q62bdtKkjw9PdWxY0clJibKx8dH3t7eSkxMVFhYmHXjhpCQEEVHR2vatGkaMGCAJOn9999Xs2bNCrxBAwAAAACUhOt6UGxBunXrpqysLCUkJOj3339X3bp1FR8fb91RTpL69OkjV1dXTZo0yXpQ7IgRI+Ti8r+7jg8dOlQzZszQuHHjJEnNmzdXv379SrpcAAAAALDYjPIwUbAQp06dcrr9980iZ0DXsi7hT8f1g8VlXQJKkM1mU1BQkI4fP14u5ivj5kVfQ2mhr5U/fF8rvpv1+5rdbi/yTRqu6UGxAAAAAFAeEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMbsVpvHz5ci1fvlynTp2SJIWEhKhnz55q2rSpJMkwDM2fP18rVqxQenq66tWrp379+ik0NNTah8PhUGJiotatW6esrCxFRkaqf//+qlKlitUmPT1dM2fO1KZNmyRJMTExiouLk5eX13WfMAAAAABcSbFGkCpXrqxHH31U48eP1/jx4xUZGanXX39dhw8fliQtWrRIS5YsUVxcnMaPHy8/Pz+NHTtWmZmZ1j5mzZqlDRs2aNiwYRozZowuXLigCRMmKDc312ozZcoUJScnKz4+XvHx8UpOTtbUqVNL6JQBAAAAoGDFCkgxMTFq1qyZgoODFRwcrEceeUQeHh7au3evDMPQ0qVLFRsbq5YtWyosLEyDBw/WxYsXtXbtWklSRkaGVq5cqSeeeEJRUVEKDw/XkCFDdOjQIW3btk2SdOTIEW3dulWDBg1SRESEIiIiNHDgQG3ZskXHjh0r+XcAAAAAAEzFmmL3R7m5ufrhhx908eJFRURE6OTJk0pNTVWTJk2sNna7XQ0bNtTu3bvVqVMnHThwQDk5OYqKirLaVK5cWWFhYdqzZ4+io6O1Z88eeXp6ql69elabiIgIeXp6avfu3QoODi6wHofDIYfDYb222WyqWLGi9Wf8+fE5li95nyefK240+hpKC30NKB/9v9gB6dChQ4qPj5fD4ZCHh4eef/55hYSEaPfu3ZIkX19fp/a+vr5KSUmRJKWmpsrNzU3e3t752qSmplptLt/H5W0KkpSUpAULFlivw8PDNXHiRFWtWrW4p1gqDpd1AX9CQUFBZV0CboDAwMCyLgG3CPoaSgt9rfzg+1rxlYfva8UOSMHBwXrjjTf0+++/a/369XrnnXc0evRoa/3lqdEwjKvus6htCkuksbGx6tKlS746Tp06pezs7KvuHze/48ePl3UJKEE2m02BgYE6ceJEkf4OAK4VfQ2lhb4G3Lzf19zc3Io8cFLsgOTm5mb9ZqROnTrav3+/li5dqm7dukm6NALk7+9vtU9LS7NGhPz8/JSdna309HSnUaS0tDTVr1/fanPu3Ll8x/3jfgpit9tlt9sLXMdfUuUDn2P5ZBgGny1KBX0NpYW+hltZeej71/0cJMMw5HA4VK1aNfn5+Vk3W5Ck7Oxs7dy50wo/tWvXlqurq1Obs2fP6tChQ4qIiJB06XqjjIwM7du3z2qzd+9eZWRkWPsBAAAAgBuhWCNIn3zyiZo2baoqVarowoULWrdunXbs2KH4+HjZbDZ17txZSUlJCgoKUmBgoJKSkuTu7q62bdtKkjw9PdWxY0clJibKx8dH3t7eSkxMVFhYmHXjhpCQEEVHR2vatGkaMGCAJOn999+37p4HAAAAADdKsQLSuXPn9Pbbb+vs2bPy9PRUzZo1FR8fb4Wbbt26KSsrSwkJCfr9999Vt25dxcfHW3eTk6Q+ffrI1dVVkyZNsh4UO2LECLm4/G8wa+jQoZoxY4bGjRsnSWrevLn69etXEucLAAAAAFdkM8rDRMFCnDp1yun23zeLnAFdy7qEPx3XDxaXdQkoQTabTUFBQTp+/Hi5mK+Mmxd9DaWFvlb+8H2t+G7W72t2u73IN2m47muQAAAAAKC8ICABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYHIrTuOkpCRt2LBBR48eVYUKFRQREaHHHntMwcHBVhvDMDR//nytWLFC6enpqlevnvr166fQ0FCrjcPhUGJiotatW6esrCxFRkaqf//+qlKlitUmPT1dM2fO1KZNmyRJMTExiouLk5eX1/WeMwAAAAAUqFgjSDt37tS9996rcePG6eWXX1Zubq7Gjh2rCxcuWG0WLVqkJUuWKC4uTuPHj5efn5/Gjh2rzMxMq82sWbO0YcMGDRs2TGPGjNGFCxc0YcIE5ebmWm2mTJmi5ORkxcfHKz4+XsnJyZo6dWoJnDIAAAAAFKxYASk+Pl4dOnRQaGioatWqpaeeekopKSk6cOCApEujR0uXLlVsbKxatmypsLAwDR48WBcvXtTatWslSRkZGVq5cqWeeOIJRUVFKTw8XEOGDNGhQ4e0bds2SdKRI0e0detWDRo0SBEREYqIiNDAgQO1ZcsWHTt2rITfAgAAAAC4pFhT7C6XkZEhSfL29pYknTx5UqmpqWrSpInVxm63q2HDhtq9e7c6deqkAwcOKCcnR1FRUVabypUrKywsTHv27FF0dLT27NkjT09P1atXz2oTEREhT09P7d6922lKXx6HwyGHw2G9ttlsqlixovVn/PnxOZYveZ8nnytuNPoaSgt9DSgf/f+aA5JhGJo9e7Zuu+02hYWFSZJSU1MlSb6+vk5tfX19lZKSYrVxc3OzQtUf2+Rtn5qamm8fl7e5XFJSkhYsWGC9Dg8P18SJE1W1atVrOb0b7nBZF/AnFBQUVNYl4AYIDAws6xJwi6CvobTQ18oPvq8VX3n4vnbNAWn69Ok6dOiQxowZk2/d5cnRMIyr7q+oba6USmNjY9WlS5d8NZw6dUrZ2dlX3TdufsePHy/rElCCbDabAgMDdeLEiSL9/APXir5W/mT3/7+yLuFPxy3h87IuAbeIm/X7mpubW5EHTq4pIM2YMUObN2/W6NGjne485+fnJ+nSCJC/v7+1PC0tzRoR8vPzU3Z2ttLT051GkdLS0lS/fn2rzblz5/Id94/7uZzdbpfdbi9wHf8glg98juWTYRh8tigV9DXcyuj7KC3loa8V6yYNhmFo+vTpWr9+vV599VVVq1bNaX21atXk5+dn3WxBkrKzs7Vz504r/NSuXVuurq5Obc6ePatDhw4pIiJC0qXrjTIyMrRv3z6rzd69e5WRkWHtBwAAAABKWrFGkKZPn661a9dq+PDhqlixonU9kKenpypUqCCbzabOnTsrKSlJQUFBCgwMVFJSktzd3dW2bVurbceOHZWYmCgfHx95e3srMTFRYWFh1o0bQkJCFB0drWnTpmnAgAGSpPfff1/NmjUr8AYNAAAAAFASihWQli9fLkkaNWqU0/KnnnpKHTp0kCR169ZNWVlZSkhI0O+//666desqPj7euqOcJPXp00eurq6aNGmS9aDYESNGyMXlfwNaQ4cO1YwZMzRu3DhJUvPmzdWvX79rOUcAQCnIGdC1rEu4opv1QmvXDxaXdQkAgMsUKyDNmzfvqm1sNpt69eqlXr16XbFNhQoVFBcXp7i4uCu28fb21tChQ4tTHgAAAABcl2JdgwQAAAAA5RkBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMDkVtYFALixcgZ0LesSruhwWRdwBa4fLC7rEgAAQBlhBAkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwERAAgAAAAATAQkAAAAATAQkAAAAADARkAAAAADA5FbcDXbu3KnFixfr4MGDOnv2rJ5//nm1aNHCWm8YhubPn68VK1YoPT1d9erVU79+/RQaGmq1cTgcSkxM1Lp165SVlaXIyEj1799fVapUsdqkp6dr5syZ2rRpkyQpJiZGcXFx8vLyup7zBQAAAIArKvYI0sWLF1WrVi3FxcUVuH7RokVasmSJ4uLiNH78ePn5+Wns2LHKzMy02syaNUsbNmzQsGHDNGbMGF24cEETJkxQbm6u1WbKlClKTk5WfHy84uPjlZycrKlTp17DKQIAAABA0RQ7IDVt2lQPP/ywWrZsmW+dYRhaunSpYmNj1bJlS4WFhWnw4MG6ePGi1q5dK0nKyMjQypUr9cQTTygqKkrh4eEaMmSIDh06pG3btkmSjhw5oq1bt2rQoEGKiIhQRESEBg4cqC1btujYsWPXecoAAAAAULBiT7ErzMmTJ5WamqomTZpYy+x2uxo2bKjdu3erU6dOOnDggHJychQVFWW1qVy5ssLCwrRnzx5FR0drz5498vT0VL169aw2ERER8vT01O7duxUcHJzv2A6HQw6Hw3pts9lUsWJF68/48+NzRGmhr6G00NdQWuhrKC3loa+VaEBKTU2VJPn6+jot9/X1VUpKitXGzc1N3t7e+drkbZ+amppvH5e3uVxSUpIWLFhgvQ4PD9fEiRNVtWrVazybG+twWRfwJxQUFFTWJfwp0deKj752behrxUdfuzb0teKjr10b+lrxlYe+VqIBKc/lydEwjKtuU9Q2V0qlsbGx6tKlS74aTp06pezs7KvuGze/48ePl3UJuEXQ11Ba6GsoLfQ1lJabta+5ubkVeeCkRAOSn5+fpEsjQP7+/tbytLQ0a0TIz89P2dnZSk9PdxpFSktLU/369a02586dy7f/P+7ncna7XXa7vcB1RQlfuPnxOaK00NdQWuhrKC30NZSW8tDXSvQ5SNWqVZOfn591swVJys7O1s6dO63wU7t2bbm6ujq1OXv2rA4dOqSIiAhJl643ysjI0L59+6w2e/fuVUZGhrUfAAAAAChpxR5BunDhgk6cOGG9PnnypJKTk+Xt7a2AgAB17txZSUlJCgoKUmBgoJKSkuTu7q62bdtKkjw9PdWxY0clJibKx8dH3t7eSkxMVFhYmHXjhpCQEEVHR2vatGkaMGCAJOn9999Xs2bNCrxBAwAAAACUhGIHpP3792v06NHW6w8//FCS1L59ew0ePFjdunVTVlaWEhIS9Pvvv6tu3bqKj4+37ignSX369JGrq6smTZpkPSh2xIgRcnH534DW0KFDNWPGDI0bN06S1Lx5c/Xr1++aTxQAAAAArqbYAalRo0aaN2/eFdfbbDb16tVLvXr1umKbChUqKC4u7ooPm5Ukb29vDR06tLjlAQAAAMA1K9FrkAAAAADgz4yABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAICJgAQAAAAAJgISAAAAAJgISAAAAABgIiABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAAGAiIAEAAACAiYAEAAAAACYCEgAAAACYCEgAAAAAYCIgAQAAAIDJrawLuJqvvvpKixcvVmpqqkJCQtS3b181aNCgrMsCAAAAUA7d1CNI33//vWbNmqUHH3xQEydOVIMGDfTaa68pJSWlrEsDAAAAUA7d1AHpiy++UMeOHXXXXXdZo0cBAQFavnx5WZcGAAAAoBy6aafYZWdn68CBA+revbvT8qioKO3evTtfe4fDIYfDYb222WyqWLGi3NxuzlN0qVO/rEv403G128u6hD8l+lrx0deuDX2t+Ohr14a+Vnz0tWtDXyu+m7WvFScT3JzpQVJaWppyc3Pl6+vrtNzX11epqan52iclJWnBggXW6zZt2mjYsGHy9/e/0aVemykfl3UFuFXQ11Ba6GsoLfQ1lBb62i3ppp5iJ10aCSrKstjYWM2aNcv6b8CAAU4jSiiazMxMjRgxQpmZmWVdCso5+hpKC30NpYW+htJCX7uxbtoRpEqVKsnFxSXfaNG5c+fyjSpJkt1ul/0mHdL7MzEMQwcPHpRhGGVdCso5+hpKC30NpYW+htJCX7uxbtoRJDc3N9WuXVvbtm1zWr5t2zbVr898UAAAAAAl76YdQZKkLl26aOrUqapdu7YiIiL0zTffKCUlRZ06dSrr0gAAAACUQzd1QGrdurXOnz+vTz/9VGfPnlVoaKj+/ve/q2rVqmVdWrllt9vVs2dPpivihqOvobTQ11Ba6GsoLfS1G8tmMHkRAAAAACTdxNcgAQAAAEBpIyABAAAAgImABAAAAAAmAhIAAAAAmAhIAAAAJYD7XgHlAwEJAACgBDz66KM6cuRIWZcB4Drd1M9BQtlKSUnRvHnz9NRTT5V1KSgHsrKydODAAXl7eyskJCTfuh9++EHt27cvo+pQnhw5ckR79+5VRESEatSooaNHj2rp0qVyOBxq166dIiMjy7pE/MnNnj27wOW5ubn67LPP5OPjI0nq06dPaZaFW0R6erq+/fZbHT9+XP7+/mrfvr0CAgLKuqxyhYCEK8r7ASQg4XodO3ZM48aNU0pKiiSpQYMGGjZsmPz9/SVJGRkZ+ve//01AwnXbunWrXn/9dXl4eOjixYt64YUX9Pbbb6tmzZqSpHHjxik+Pp6QhOuydOlS1axZU15eXvnWHT16VB4eHmVQFcqrgQMH6p///Kd8fHx08uRJvfzyy5KksLAwbd68WZ9//rnGjRunGjVqlHGl5QcB6Ra2adOmQtf/9ttvpVQJyruPP/5YoaGhGj9+vDIyMjR79my98sorGjVqFL/1QolasGCBunbtqocffljr1q3T5MmTdc899+iRRx6RJM2ZM0efffYZAQnX5eGHH9aKFSv0xBNPOPWlRx55RIMHD843Sg5cj9TUVOXm5kqSPvnkE9WoUUMvvvii3N3d5XA49K9//Utz587Vs88+W8aVlh8EpFvYG2+8UdYl4BaxZ88evfLKK6pUqZIqVaqkESNGKCEhQa+++qpGjhwpd3f3si4R5cThw4f19NNPS5JatWqlt99+Wy1btrTWt2nTRitXriyr8lBOxMbGqnHjxpo6daqaN2+uRx99VG5ufKXCjbdv3z4NGjTI+nfTbrerR48eevPNN8u4svKFn+ZbmJ+fn/r166cWLVoUuD45OVkjRowo5apQHmVlZcnFxfmeMP3799eMGTM0atQoDR06tIwqQ3nm4uIiu93uNA2qYsWKysjIKMOqUF7UrVtXEydOVEJCgl588UX+HsMNZbPZJEkOh0O+vr5O63x9fZWWllYWZZVbBKRbWO3atXXw4MErBiSgpAQHB+vAgQP5pp3ExcXJMAy9/vrrZVQZyptq1arpxIkTCgwMlCSNHTvWaRrn6dOnrWvfgOvl4eGhp59+WuvWrdM//vEPaxoUUNLGjBkjV1dXZWZm6vjx4woNDbXWpaSkWDcGQckgIN3CunbtqosXL15xfWBgoEaOHFmKFaG8atGihdatW6d27drlW9evXz8ZhqGvv/66DCpDedOpUyenL6lhYWFO63/66SeuP0KJa9OmjW677TYdOHCA6ypR4nr27On0ukKFCk6vN2/erNtuu600Syr3bAZPNQMAAAAASTwoFgAAAAAsBCQAAAAAMBGQAAAAAMDETRoAAE5Wr16tf//739ZrFxcXVapUSQ0bNtRDDz2koKCgG37st99+W9WqVbthxymKrVu36osvvtDhw4d1/vx5+fj4KCgoSNHR0erevbvVbuHChQoJCeGOoABQThCQAAAFeuqppxQcHCyHw6Hdu3dr4cKF2rFjhyZNmiRvb+8bcsxmzZpp7NixZX4r7uXLlyshIUEtW7ZUXFycvL29dfr0ae3evVs//vijU0BKSkrSHXfcQUACgHKCgAQAKFBoaKjq1KkjSWrUqJFyc3M1b948bdy4UXfeeecNOWalSpVUqVKlG7Lv4vjss8/UoEEDPffcc07L27VrVyrPusnNzVVOTo7sdvsNPxYAwBkBCQBQJLVr15YknTt3zmn5/v37tWDBAu3atUtZWVmqUaOGunfvrtatW0uSkpOTNXz4cA0aNEgdO3Z02vann37S+PHjNXz4cMXExFxxit22bdv02Wefaf/+/crJyVF4eLh69eqlxo0bS5IOHz6s5557Ts8884xatWolSTpw4IBefPFFhYSE6M0337T2NXHiRJ05c0YTJ0684rmeP39eERERBa5zcfnf5bu9evWSJH377bf69ttvJUkNGzbUqFGjlJaWpnnz5mnHjh1KSUmRu7u7QkND1atXLzVo0MDax8mTJ/X000+rd+/eys7O1sqVK3X69Gm9+OKLioqKUlJSkr777julpKTIbrcrICBAHTt2VOfOna9YPwDg2hGQAABFcvLkSUlyugbpv//9r1577TXVq1dPAwYMkKenp77//nu99dZbysrKUocOHVSrVi2Fh4dr1apV+QLS6tWr5evrq6ZNm17xuN99953eeecdxcTEaPDgwXJ1ddXXX3+tcePGKT4+Xo0bN1ZoaKj8/f21fft2KyBt27ZNFSpU0JEjR3TmzBlVrlxZOTk52rlzpzp16lTouUZERGj9+vWaN2+eWrRoobCwMKdglGfs2LEaM2aMGjVqpB49ekiSPD09JUnp6emSpL/+9a/y8/PThQsXtGHDBo0aNUqvvvqqGjVq5LSvL7/8UkFBQXr88cfl6empwMBALV68WPPnz9eDDz6ohg0bKjs7W8eOHdPvv/9eaP0AgGtHQAIAFChvmpfD4dCuXbu0cOFCNWjQQDExMVab6dOnKzQ0VK+++qpcXV0lSdHR0UpLS9OcOXPUrl07ubi4qEOHDpo5c6aOHTum4OBgSZcCxKZNm3Tvvfda217u4sWLmjVrlpo1a6YXXnjBWt60aVONGDFCc+bMsUaRIiMjtX37dqvN9u3b9Ze//EU//vijtm/frvbt22vfvn3KzMxUVFRUoec+YMAAvfHGG1qwYIEWLFigChUqqH79+rr99tt19913y83t0j+fERERstlsqlSpUr4Rp+DgYPXv39/p/WzSpIlOnTqlL7/8Ml9Astvtio+Pt/YtSbt27VJYWJg1UpX3/gIAbhwCEgCgQPHx8U6va9SooeHDh1th5sSJEzp69Kgef/xxSVJOTo7VtlmzZtqyZYuOHTumkJAQ/eUvf9FHH32k1atX69FHH5UkrVu3Tg6Ho9DrmXbv3q309HS1b9/eaf/SpaCwePFiXbhwQR4eHmrcuLHWrFmjkydPys/PT7t27dI999yj8+fPa9u2bWrfvr22b98uu92u2267rdBzDwwM1BtvvKFdu3Zp586d2r9/v3bu3Knt27dr9erVGjNmjCpUqHDV93D58uVasWKFjhw5IofD4fReXi4mJsYpHElS3bp1NX/+fCUkJCgmJkYRERHWCBUA4MYgIAEACvT000+rRo0ayszM1Pfff69vvvlGkydP1ksvvSRJSk1NlSQlJiYqMTGxwH2cP39ekuTt7a3mzZvru+++08MPPywXFxetXr1adevWVWho6BVryLve6Y/XEF0uPT3dCkjSpal11apVU05OjiIjI3Xu3Dl9+umnki6NKtWvX79I4cbFxUUNGzZUw4YNJUkXLlzQe++9p++//16rVq3SvffeW+j2X3zxhT788EN16tRJDz30kHx8fOTi4qK5c+fq6NGj+doXdOe+2NhYeXh4aM2aNfr666/l4uKiBg0aqHfv3tYNNAAAJYuABAAoUI0aNawv4ZGRkcrNzdXKlSv1448/6o477rDuNte9e3e1bNmywH3kTaeTpDvvvFM//vijtm3bpoCAAO3fv99pClpBfHx8JElxcXGqV69egW38/PwkSVWqVFFQUJC2b9+uqlWrqk6dOvLy8lJkZKQSEhK0d+9e7dmzx2m6WnF4eHioe/fu+v7773X48OGrtl+zZo0aNWqkAQMGOC3PzMws8jFdXV3VpUsXdenSRb///ru2b9+uOXPmaNy4cXr33Xfl7u5e7PMAABSOgAQAKJLHHntM69ev19y5c9WiRQsFBwcrKChIv/76qzVtrjBNmjRR5cqVtWrVKgUEBMhut6tt27aFbnPbbbfJy8tLR44c0X333XfVYzRu3Fg//PCDAgICrBs/BAcHKyAgQPPmzVNOTo410lSYs2fPFjiikzfy88d1drtdWVlZBe7n8ilzv/76q/bs2aOAgICr1nA5Ly8v3XHHHTpz5oxmzZqlU6dOKSQkpNj7AQAUjoAEACgSb29vxcbG6qOPPtLatWvVrl07DRgwQOPHj9e4cePUvn17Va5cWenp6Tp69KgOHjyoZ5991trexcVF7dq105IlS1SxYkW1bNnyqtfTeHh46Mknn9Q777yj9PR0a+QqLS1Nv/76q9LS0pxGaBo3bqzly5fr/Pnz6tOnj7U8MjJSq1evlpeXl3W78sI8++yzaty4saKjoxUYGKisrCzt27dPn3/+uXx9fZ3uxhcWFqadO3dq06ZN8vf3V8WKFRUcHKzmzZvr008/1bx589SwYUMdO3ZMCxYsULVq1Yr8LKUJEyYoLCxMtWvXVqVKlZSSkqIlS5aoatWqCgwMLNI+AADFQ0ACABTZfffdp2XLlunTTz9V27ZtFRkZqddee00LFy7U7NmzlZ6eLh8fH4WEhFi32/6jO++8U5999pkcDoc6dOhQpGO2a9dOAQEBWrx4sd5//31lZmbK19dXtWrVyrePyMhI2Ww2VahQwemuclFRUVq9erUaNWpU4O26L9e7d29t3bpVSUlJSk1NVU5OjqpUqaK2bdvqwQcfdBpB6tu3r6ZPn67Jkyfr4sWL1nOQHnzwQV28eFErV67UokWLFBISogEDBmjDhg3auXNnkc49MjJS69ev14oVK5SZmSk/Pz9FRUWpR48e+UanAAAlw2YYhlHWRQAAAADAzeDqv0YDAAAAgFsEAQkAAAAATAQkAAAAADARkAAAAADAREACAAAAABMBCQAAAABMBCQAAAAAMBGQAAAAAMBEQAIAAAAAEwEJAAAAAEwEJAAAAAAwEZAAAAAAwPT/AYKqYYZzwqYpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = df[\"ratingvalue\"].value_counts().sort_index().plot(kind=\"bar\", title=\"Count of Reviews by Stars\", figsize=(10,5))\n",
    "\n",
    "ax.set_xlabel('Review Stars')\n",
    "plt.savefig('stars.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Roberta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL = f'cardiffnlp/twitter-roberta-base-sentiment'\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(MODEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def polarity_scores_roberta(example):\n",
    "  encoded_text = tokenizer(example, return_tensors = 'pt')\n",
    "  output = model(**encoded_text)\n",
    "  scores = output[0][0].detach().numpy()\n",
    "  scores = softmax(scores)\n",
    "  scores_dict = {\n",
    "    'roberta_neg' : scores[0],\n",
    "    'roberta_neu' : scores[1],\n",
    "    'roberta_pos' : scores[2]\n",
    "  }\n",
    "  return scores_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f812a35379d942eab17a90ce58fc4541",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15017 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = {}\n",
    "vader_result_rename = {}\n",
    "for i,row in tqdm(df.iterrows(), total = len(df)):\n",
    "  text = row['text']\n",
    "  myid = i\n",
    "  vader_result = sia.polarity_scores(text)\n",
    "  for key,value in vader_result.items():\n",
    "    vader_result_rename[f'vader_{key}'] = value\n",
    "  roberta_result = polarity_scores_roberta(text)\n",
    "  both = {**vader_result_rename, **roberta_result}\n",
    "  res[myid] = both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame(res).T\n",
    "results_df= results_df.reset_index().rename(columns={\"index\":\"id\"})\n",
    "results_df = results_df.merge(df, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vader_neg</th>\n",
       "      <th>vader_neu</th>\n",
       "      <th>vader_pos</th>\n",
       "      <th>vader_compound</th>\n",
       "      <th>roberta_neg</th>\n",
       "      <th>roberta_neu</th>\n",
       "      <th>roberta_pos</th>\n",
       "      <th>createdAt</th>\n",
       "      <th>text</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>ratingvalue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.008115</td>\n",
       "      <td>0.804791</td>\n",
       "      <td>0.187094</td>\n",
       "      <td>2022-02-04T11:10:14.474Z</td>\n",
       "      <td>he received my order around 2.30 pm but delive...</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.833</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.3412</td>\n",
       "      <td>0.915435</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.009115</td>\n",
       "      <td>2022-07-16T05:59:24Z</td>\n",
       "      <td>taste was not that good. and the amount of cof...</td>\n",
       "      <td>Israel</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.734</td>\n",
       "      <td>0.266</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>0.041308</td>\n",
       "      <td>0.409943</td>\n",
       "      <td>0.548750</td>\n",
       "      <td>2022-04-24T12:15:02.221Z</td>\n",
       "      <td>It is better if the girl is a little bigger</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.339</td>\n",
       "      <td>0.661</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.5563</td>\n",
       "      <td>0.953995</td>\n",
       "      <td>0.041070</td>\n",
       "      <td>0.004935</td>\n",
       "      <td>2021-11-16T15:55:42.189Z</td>\n",
       "      <td>take too much time. today i feel very angry</td>\n",
       "      <td>Zerin Ibnat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.374</td>\n",
       "      <td>0.626</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.8386</td>\n",
       "      <td>0.963817</td>\n",
       "      <td>0.032379</td>\n",
       "      <td>0.003804</td>\n",
       "      <td>2022-06-30T05:53:21Z</td>\n",
       "      <td>Coleslaw was not fresh and it had a bad smell....</td>\n",
       "      <td>faria</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15012</th>\n",
       "      <td>15012</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.737</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>0.039455</td>\n",
       "      <td>0.282166</td>\n",
       "      <td>0.678379</td>\n",
       "      <td>2021-06-27T10:38:17.471Z</td>\n",
       "      <td>nice taste</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15013</th>\n",
       "      <td>15013</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.408</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>0.002754</td>\n",
       "      <td>0.041969</td>\n",
       "      <td>0.955276</td>\n",
       "      <td>2021-10-26T07:56:19.798Z</td>\n",
       "      <td>Taste is good.</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15014</th>\n",
       "      <td>15014</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.738</td>\n",
       "      <td>0.262</td>\n",
       "      <td>0.4927</td>\n",
       "      <td>0.001298</td>\n",
       "      <td>0.014188</td>\n",
       "      <td>0.984514</td>\n",
       "      <td>2022-08-18T13:13:11Z</td>\n",
       "      <td>Masha Allah the food was very good today too. ...</td>\n",
       "      <td>Mithila</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15015</th>\n",
       "      <td>15015</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.859449</td>\n",
       "      <td>0.132488</td>\n",
       "      <td>0.008062</td>\n",
       "      <td>2021-10-30T09:29:16.450Z</td>\n",
       "      <td>Chicken was undercooked</td>\n",
       "      <td>Towsif Al Hafiz</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15016</th>\n",
       "      <td>15016</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.295514</td>\n",
       "      <td>0.669402</td>\n",
       "      <td>0.035084</td>\n",
       "      <td>2022-01-18T08:51:39.950Z</td>\n",
       "      <td>I ordered Mehjan set mill Gave me khichei mila...</td>\n",
       "      <td>ashik mahmud</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15017 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vader_neg  vader_neu  vader_pos  vader_compound  roberta_neg  \\\n",
       "0          0      0.000      1.000      0.000          0.0000     0.008115   \n",
       "1          1      0.167      0.833      0.000         -0.3412     0.915435   \n",
       "2          2      0.000      0.734      0.266          0.4404     0.041308   \n",
       "3          3      0.339      0.661      0.000         -0.5563     0.953995   \n",
       "4          4      0.374      0.626      0.000         -0.8386     0.963817   \n",
       "...      ...        ...        ...        ...             ...          ...   \n",
       "15012  15012      0.000      0.263      0.737          0.4215     0.039455   \n",
       "15013  15013      0.000      0.408      0.592          0.4404     0.002754   \n",
       "15014  15014      0.000      0.738      0.262          0.4927     0.001298   \n",
       "15015  15015      0.000      1.000      0.000          0.0000     0.859449   \n",
       "15016  15016      0.000      1.000      0.000          0.0000     0.295514   \n",
       "\n",
       "       roberta_neu  roberta_pos                 createdAt  \\\n",
       "0         0.804791     0.187094  2022-02-04T11:10:14.474Z   \n",
       "1         0.075450     0.009115      2022-07-16T05:59:24Z   \n",
       "2         0.409943     0.548750  2022-04-24T12:15:02.221Z   \n",
       "3         0.041070     0.004935  2021-11-16T15:55:42.189Z   \n",
       "4         0.032379     0.003804      2022-06-30T05:53:21Z   \n",
       "...            ...          ...                       ...   \n",
       "15012     0.282166     0.678379  2021-06-27T10:38:17.471Z   \n",
       "15013     0.041969     0.955276  2021-10-26T07:56:19.798Z   \n",
       "15014     0.014188     0.984514      2022-08-18T13:13:11Z   \n",
       "15015     0.132488     0.008062  2021-10-30T09:29:16.450Z   \n",
       "15016     0.669402     0.035084  2022-01-18T08:51:39.950Z   \n",
       "\n",
       "                                                    text     reviewerName  \\\n",
       "0      he received my order around 2.30 pm but delive...        Anonymous   \n",
       "1      taste was not that good. and the amount of cof...           Israel   \n",
       "2            It is better if the girl is a little bigger        Anonymous   \n",
       "3            take too much time. today i feel very angry      Zerin Ibnat   \n",
       "4      Coleslaw was not fresh and it had a bad smell....            faria   \n",
       "...                                                  ...              ...   \n",
       "15012                                         nice taste        Anonymous   \n",
       "15013                                     Taste is good.        Anonymous   \n",
       "15014  Masha Allah the food was very good today too. ...          Mithila   \n",
       "15015                            Chicken was undercooked  Towsif Al Hafiz   \n",
       "15016  I ordered Mehjan set mill Gave me khichei mila...     ashik mahmud   \n",
       "\n",
       "       ratingvalue  \n",
       "0                4  \n",
       "1                1  \n",
       "2                5  \n",
       "3                1  \n",
       "4                1  \n",
       "...            ...  \n",
       "15012            5  \n",
       "15013            5  \n",
       "15014            5  \n",
       "15015            4  \n",
       "15016            1  \n",
       "\n",
       "[15017 rows x 12 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = 0\n",
    "for i,row in results_df.iterrows():\n",
    "  positive = row[\"roberta_pos\"]\n",
    "  negative = row[\"roberta_neg\"]\n",
    "  neutral = row[\"roberta_neu\"]\n",
    "  if(max(positive,negative,neutral) == positive and row[\"ratingvalue\"]==4 or row[\"ratingvalue\"]==5):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == neutral and row[\"ratingvalue\"]==3):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == negative and row[\"ratingvalue\"]==1 or row[\"ratingvalue\"]==2):\n",
    "    tp+=1\n",
    "\n",
    "roberta_accuracy = (tp)/(results_df.shape[0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = 0\n",
    "fp = 0 \n",
    "for i,row in results_df.iterrows():\n",
    "  positive = row[\"roberta_pos\"]\n",
    "  negative = row[\"roberta_neg\"]\n",
    "  neutral = row[\"roberta_neu\"]\n",
    "  if(max(positive,negative,neutral) == positive and row[\"ratingvalue\"]==4 or row[\"ratingvalue\"]==5):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == neutral and row[\"ratingvalue\"]==3):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == negative and row[\"ratingvalue\"]==1 or row[\"ratingvalue\"]==2):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == positive and row[\"ratingvalue\"]<4):\n",
    "    fp+=1\n",
    "roberta_precision = (tp)/(tp+fp)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = 0\n",
    "fn = 0 \n",
    "for i,row in results_df.iterrows():\n",
    "  positive = row[\"roberta_pos\"]\n",
    "  negative = row[\"roberta_neg\"]\n",
    "  neutral = row[\"roberta_neu\"]\n",
    "  if(max(positive,negative,neutral) == positive and row[\"ratingvalue\"]==4 or row[\"ratingvalue\"]==5):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == neutral and row[\"ratingvalue\"]==3):\n",
    "    tp+=1\n",
    "  elif(max(positive,negative,neutral) == negative and row[\"ratingvalue\"]==1 or row[\"ratingvalue\"]==2):\n",
    "    tp+=1\n",
    "  if(max(positive,negative,neutral) == negative and row[\"ratingvalue\"]>2):\n",
    "    fn+=1\n",
    "  if(max(positive,negative,neutral) == negative and row[\"ratingvalue\"]!=3):  \n",
    "    fn+=1\n",
    "    \n",
    "roberta_recall = (tp)/(tp+fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "roberta_f1_Score = 2 * (roberta_precision * roberta_recall) / (roberta_precision + roberta_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Roberta accuracy: 0.7366984084704002\n",
      "Roberta Precision: 0.9716318285613912\n",
      "Roberta Recall: 0.5505897576270343\n",
      "Roberta F1 Score: 0.7028812859366563\n"
     ]
    }
   ],
   "source": [
    "print(\"Roberta accuracy: \" + str(roberta_accuracy))\n",
    "print(\"Roberta Precision: \" + str(roberta_precision))\n",
    "print(\"Roberta Recall: \" + str(roberta_recall))\n",
    "print(\"Roberta F1 Score: \" + str(roberta_f1_Score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DistilBERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfaf7fea49754ceca1c85bd777b95f5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/629 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d99fc07e86ef4a60805410295819d700",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/268M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7314e10c83d4206bc9513685a9b75c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b0adf8353844ed089e06b793e57258c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "sent_pipeline = pipeline(\"sentiment-analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da8bdb60d5ce4710a5a37bf662640027",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15017 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = {}\n",
    "pipeline_result_rename = {}\n",
    "for i,row in tqdm(df.iterrows(), total = len(df)):\n",
    "    text = row['text']\n",
    "    myid = i\n",
    "    pipeline_result = sent_pipeline(text)\n",
    "    for key,value in pipeline_result[0].items():\n",
    "        pipeline_result_rename[f'pipeline_{key}'] = value\n",
    "    res[myid] = pipeline_result_rename.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df2 = pd.DataFrame(res).T\n",
    "results_df2= results_df2.reset_index().rename(columns={\"index\":\"id\"})\n",
    "results_df2 = results_df2.merge(df, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_reviews = results_df2[results_df2.pipeline_label == 'POSITIVE']\n",
    "pos_text = \" \".join(review for review in pos_reviews.text)\n",
    "neg_reviews = results_df2[results_df2.pipeline_label == 'NEGATIVE']\n",
    "neg_text = \" \".join(review for review in neg_reviews.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "neg_reviews = neg_reviews[neg_reviews.ratingvalue <4]\n",
    "pos_reviews = pos_reviews[pos_reviews.ratingvalue >3]\n",
    "distilbert_accuracy = (neg_reviews.shape[0] + pos_reviews.shape[0]) / results_df2.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "neg_reviews = results_df2[results_df2.pipeline_label == 'NEGATIVE']\n",
    "pos_reviews = results_df2[results_df2.pipeline_label == 'POSITIVE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision_pos_reviews = pos_reviews[pos_reviews.ratingvalue <4]\n",
    "\n",
    "distilbert_precision = pos_reviews.shape[0]/(precision_pos_reviews.shape[0]+pos_reviews.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "recall_neg_reviews = neg_reviews[neg_reviews.ratingvalue >3]\n",
    "distilbert_recall = (neg_reviews.shape[0] + pos_reviews.shape[0])/(recall_neg_reviews.shape[0]+(neg_reviews.shape[0] + pos_reviews.shape[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "distilbert_f1_Score = 2 * (distilbert_precision * distilbert_recall) / (distilbert_precision + distilbert_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DistilBERT accuracy: 0.7673969501231938\n",
      "DistilBERT Precision: 0.8289807762060211\n",
      "DistilBERT Recall: 0.8548414641088404\n",
      "DistilBERT F1 Score: 0.8417125317426934\n"
     ]
    }
   ],
   "source": [
    "print(\"DistilBERT accuracy: \" + str(distilbert_accuracy))\n",
    "print(\"DistilBERT Precision: \" + str(distilbert_precision))\n",
    "print(\"DistilBERT Recall: \" + str(distilbert_recall))\n",
    "print(\"DistilBERT F1 Score: \" + str(distilbert_f1_Score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_pipeline = pipeline(\"sentiment-analysis\", model=\"nlptown/bert-base-multilingual-uncased-sentiment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "498e402174aa4adfaf297a1175080571",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15017 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = {}\n",
    "pipeline_result_rename = {}\n",
    "for i,row in tqdm(df.iterrows(), total = len(df)):\n",
    "    text = row['text']\n",
    "    myid = i\n",
    "    pipeline_result = sent_pipeline(text)\n",
    "    for key,value in pipeline_result[0].items():\n",
    "        pipeline_result_rename[f'pipeline_{key}'] = value\n",
    "    res[myid] = pipeline_result_rename.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df3 = pd.DataFrame(res).T\n",
    "results_df3= results_df3.reset_index().rename(columns={\"index\":\"id\"})\n",
    "results_df3 = results_df3.merge(df, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pipeline_label</th>\n",
       "      <th>pipeline_score</th>\n",
       "      <th>createdAt</th>\n",
       "      <th>text</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>ratingvalue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1 star</td>\n",
       "      <td>0.507326</td>\n",
       "      <td>2022-02-04T11:10:14.474Z</td>\n",
       "      <td>he received my order around 2.30 pm but delive...</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2 stars</td>\n",
       "      <td>0.561847</td>\n",
       "      <td>2022-07-16T05:59:24Z</td>\n",
       "      <td>taste was not that good. and the amount of cof...</td>\n",
       "      <td>Israel</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3 stars</td>\n",
       "      <td>0.577202</td>\n",
       "      <td>2022-04-24T12:15:02.221Z</td>\n",
       "      <td>It is better if the girl is a little bigger</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2 stars</td>\n",
       "      <td>0.447705</td>\n",
       "      <td>2021-11-16T15:55:42.189Z</td>\n",
       "      <td>take too much time. today i feel very angry</td>\n",
       "      <td>Zerin Ibnat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1 star</td>\n",
       "      <td>0.451526</td>\n",
       "      <td>2022-06-30T05:53:21Z</td>\n",
       "      <td>Coleslaw was not fresh and it had a bad smell....</td>\n",
       "      <td>faria</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15012</th>\n",
       "      <td>15012</td>\n",
       "      <td>4 stars</td>\n",
       "      <td>0.473838</td>\n",
       "      <td>2021-06-27T10:38:17.471Z</td>\n",
       "      <td>nice taste</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15013</th>\n",
       "      <td>15013</td>\n",
       "      <td>4 stars</td>\n",
       "      <td>0.509026</td>\n",
       "      <td>2021-10-26T07:56:19.798Z</td>\n",
       "      <td>Taste is good.</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15014</th>\n",
       "      <td>15014</td>\n",
       "      <td>5 stars</td>\n",
       "      <td>0.620138</td>\n",
       "      <td>2022-08-18T13:13:11Z</td>\n",
       "      <td>Masha Allah the food was very good today too. ...</td>\n",
       "      <td>Mithila</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15015</th>\n",
       "      <td>15015</td>\n",
       "      <td>1 star</td>\n",
       "      <td>0.410384</td>\n",
       "      <td>2021-10-30T09:29:16.450Z</td>\n",
       "      <td>Chicken was undercooked</td>\n",
       "      <td>Towsif Al Hafiz</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15016</th>\n",
       "      <td>15016</td>\n",
       "      <td>1 star</td>\n",
       "      <td>0.685167</td>\n",
       "      <td>2022-01-18T08:51:39.950Z</td>\n",
       "      <td>I ordered Mehjan set mill Gave me khichei mila...</td>\n",
       "      <td>ashik mahmud</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15017 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id pipeline_label pipeline_score                 createdAt  \\\n",
       "0          0         1 star       0.507326  2022-02-04T11:10:14.474Z   \n",
       "1          1        2 stars       0.561847      2022-07-16T05:59:24Z   \n",
       "2          2        3 stars       0.577202  2022-04-24T12:15:02.221Z   \n",
       "3          3        2 stars       0.447705  2021-11-16T15:55:42.189Z   \n",
       "4          4         1 star       0.451526      2022-06-30T05:53:21Z   \n",
       "...      ...            ...            ...                       ...   \n",
       "15012  15012        4 stars       0.473838  2021-06-27T10:38:17.471Z   \n",
       "15013  15013        4 stars       0.509026  2021-10-26T07:56:19.798Z   \n",
       "15014  15014        5 stars       0.620138      2022-08-18T13:13:11Z   \n",
       "15015  15015         1 star       0.410384  2021-10-30T09:29:16.450Z   \n",
       "15016  15016         1 star       0.685167  2022-01-18T08:51:39.950Z   \n",
       "\n",
       "                                                    text     reviewerName  \\\n",
       "0      he received my order around 2.30 pm but delive...        Anonymous   \n",
       "1      taste was not that good. and the amount of cof...           Israel   \n",
       "2            It is better if the girl is a little bigger        Anonymous   \n",
       "3            take too much time. today i feel very angry      Zerin Ibnat   \n",
       "4      Coleslaw was not fresh and it had a bad smell....            faria   \n",
       "...                                                  ...              ...   \n",
       "15012                                         nice taste        Anonymous   \n",
       "15013                                     Taste is good.        Anonymous   \n",
       "15014  Masha Allah the food was very good today too. ...          Mithila   \n",
       "15015                            Chicken was undercooked  Towsif Al Hafiz   \n",
       "15016  I ordered Mehjan set mill Gave me khichei mila...     ashik mahmud   \n",
       "\n",
       "       ratingvalue  \n",
       "0                4  \n",
       "1                1  \n",
       "2                5  \n",
       "3                1  \n",
       "4                1  \n",
       "...            ...  \n",
       "15012            5  \n",
       "15013            5  \n",
       "15014            5  \n",
       "15015            4  \n",
       "15016            1  \n",
       "\n",
       "[15017 rows x 7 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_reviews = results_df3[(results_df3.pipeline_label == \"4 stars\") | (results_df3.pipeline_label == \"5 stars\")]\n",
    "neg_reviews = results_df3[(results_df3.pipeline_label == \"3 stars\") | (results_df3.pipeline_label == \"2 stars\") | (results_df3.pipeline_label == \"1 star\")]\n",
    "pos_reviews.shape\n",
    "neg_reviews.shape\n",
    "\n",
    "pos_reviews = pos_reviews[pos_reviews.ratingvalue>3]\n",
    "neg_reviews = neg_reviews[neg_reviews.ratingvalue<4]\n",
    "bert_accuracy = (neg_reviews.shape[0] + pos_reviews.shape[0]) / results_df3.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_reviews = results_df3[(results_df3.pipeline_label == \"4 stars\") | (results_df3.pipeline_label == \"5 stars\")]\n",
    "neg_reviews = results_df3[(results_df3.pipeline_label == \"3 stars\") | (results_df3.pipeline_label == \"2 stars\") | (results_df3.pipeline_label == \"1 star\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision_pos_reviews = pos_reviews[pos_reviews.ratingvalue <4]\n",
    "precision_pos_reviews.shape\n",
    "bert_precision = pos_reviews.shape[0]/(precision_pos_reviews.shape[0]+pos_reviews.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "recall_neg_reviews = neg_reviews[neg_reviews.ratingvalue >3]\n",
    "bert_recall = (neg_reviews.shape[0] + pos_reviews.shape[0])/(recall_neg_reviews.shape[0]+(neg_reviews.shape[0] + pos_reviews.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_f1_score = 2 * (bert_precision * bert_recall) / (bert_precision+bert_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BERT accuracy: 0.7585403209695678\n",
      "BERT Precision: 0.8483554036736437\n",
      "BERT Recall: 0.8373947471142587\n",
      "BERT F1 Score: 0.8428394425950585\n"
     ]
    }
   ],
   "source": [
    "print(\"BERT accuracy: \" + str(bert_accuracy))\n",
    "print(\"BERT Precision: \" + str(bert_precision))\n",
    "print(\"BERT Recall: \" + str(bert_recall))\n",
    "print(\"BERT F1 Score: \" + str(bert_f1_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Afinn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: afinn in c:\\programdata\\anaconda3\\lib\\site-packages (0.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install afinn\n",
    "from afinn import Afinn\n",
    "\n",
    "# Initialize the Afinn object\n",
    "afinn = Afinn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b97f9fb14f2047c38c32459c14ca511e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15017 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = {}\n",
    "pipeline_result_rename = {}\n",
    "for i,row in tqdm(df.iterrows(), total = len(df)):\n",
    "    text = row['text']\n",
    "    myid = i\n",
    "    value = afinn.score(text)\n",
    "    pipeline_result_rename[\"score\"] = value\n",
    "    res[myid] = pipeline_result_rename.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df4 = pd.DataFrame(res).T\n",
    "results_df4= results_df4.reset_index().rename(columns={\"index\":\"id\"})\n",
    "results_df4 = results_df4.merge(df, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_reviews = results_df4[(results_df4.score>0)]\n",
    "neg_reviews = results_df4[(results_df4.score<=0)]\n",
    "\n",
    "pos_reviews = pos_reviews[pos_reviews.ratingvalue>3]\n",
    "neg_reviews = neg_reviews[neg_reviews.ratingvalue<4]\n",
    "afinn_accuracy = (neg_reviews.shape[0] + pos_reviews.shape[0]) / results_df4.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_reviews = results_df4[(results_df4.score>0)]\n",
    "neg_reviews = results_df4[(results_df4.score<=0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision_pos_reviews = pos_reviews[pos_reviews.ratingvalue <4]\n",
    "precision_pos_reviews.shape\n",
    "afinn_precision = pos_reviews.shape[0]/(precision_pos_reviews.shape[0]+pos_reviews.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "recall_neg_reviews = neg_reviews[neg_reviews.ratingvalue >3]\n",
    "afinn_recall = (neg_reviews.shape[0] + pos_reviews.shape[0])/(recall_neg_reviews.shape[0]+(neg_reviews.shape[0] + pos_reviews.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "afinn_f1score = 2 * (afinn_precision * afinn_recall) / (afinn_precision + afinn_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Afinn accuracy: 0.7273756409402677\n",
      "Afinn Precision: 0.7682409709021787\n",
      "Afinn Recall: 0.8568412644071665\n",
      "Afinn F1 Score: 0.8101258515719454\n"
     ]
    }
   ],
   "source": [
    "print(\"Afinn accuracy: \" + str(afinn_accuracy))\n",
    "print(\"Afinn Precision: \" + str(afinn_precision))\n",
    "print(\"Afinn Recall: \" + str(afinn_recall))\n",
    "print(\"Afinn F1 Score: \" + str(afinn_f1score))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}