Data Visualizations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = pd.read_csv('MoneyControl-First-1-10-Pages.csv')\n",
    "d2 = pd.read_csv('MoneyControl-First-11-20-Pages.csv')\n",
    "d3 = pd.read_csv('MoneyControl-First-1-10-Pages-New.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "d4 = pd.concat([d1,d2,d3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(651, 9)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d4.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2995391705069124"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(d4.Label)/len(d4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = []\n",
    "for i in d4.Label:\n",
    "    if i == 2:\n",
    "        labels.append('V Pos')\n",
    "    elif i==1:\n",
    "        labels.append('Pos')\n",
    "    elif i == 0:\n",
    "        labels.append('Neu')\n",
    "    elif i==-1:\n",
    "        labels.append('Neg')\n",
    "    elif i==-2:\n",
    "        labels.append('V Neg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdwAAAFCCAYAAABbz2zGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAEtJJREFUeJzt3X+M5HV9x/HnS06tPwv0For86KI9TamtB90ihmJoscoPI2IrcjGISD0xEEvVxFOTQk1NaStSmyr2FAqkiqAnhRRqIbQKJEVd8MRTRIGeenKBFYzSQjAH7/6x36vDucfu7cx+Zmf3+Ug2O/OZ78y8881lnvl+Z242VYUkSVpYTxn2AJIkLQcGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSAyuGPQDAypUra3x8fNhjSJK0S2699dYfVdXYXLZdFMEdHx9ncnJy2GNIkrRLknxvrtt6SlmSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNTDrHy9Isj9wKfCrwOPA+qr6SJI9gcuBcWAzcGJV/ThJgI8AxwIPA2+uqtsWZnxJy8H4umuGPcKisPnc44Y9gvowlyPcbcC7quo3gMOAM5IcBKwDbqiqVcAN3XWAY4BV3c9a4IKBTy1J0oiZNbhVtXX7EWpVPQTcAewLHA9c0m12CfDa7vLxwKU17RZg9yT7DHxySZJGyC69h5tkHDgY+DKwd1VthekoA3t1m+0L/KDnblu6NUmSlq05BzfJs4ENwFlV9dMn23SGtZrh8dYmmUwyOTU1NdcxJEkaSXMKbpKnMh3bT1XV57vl+7afKu5+39+tbwH277n7fsC9Oz5mVa2vqomqmhgbG5vv/JIkjYRZg9t96vhC4I6q+nDPTVcDp3SXTwGu6ll/U6YdBvxk+6lnSZKWq1n/WxBwOHAy8I0kG7u19wHnAlckOQ34PvD67rZrmf4vQXcx/d+CTh3oxJIkjaBZg1tVNzPz+7IAR82wfQFn9DmXJElLit80JUlSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSA7MGN8lFSe5Psqln7fIkG7ufzUk2duvjSR7pue3jCzm8JEmjYsUctrkY+Afg0u0LVfWG7ZeTnAf8pGf7u6tq9aAGlCRpKZg1uFV1Y5LxmW5LEuBE4A8GO5YkadDG110z7BEWhc3nHjeU5+33PdwjgPuq6rs9awcm+VqSLyU5Ymd3TLI2yWSSyampqT7HkCRpces3uGuAy3qubwUOqKqDgXcCn07y3JnuWFXrq2qiqibGxsb6HEOSpMVt3sFNsgJ4HXD59rWqerSqHugu3wrcDbyw3yElSRp1/RzhvgL4dlVt2b6QZCzJbt3l5wOrgHv6G1GSpNE3l/8WdBnwX8CLkmxJclp300k88XQywMuB25N8HfgccHpVPTjIgSVJGkVz+ZTymp2sv3mGtQ3Ahv7HkiRpafGbpiRJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAYMriRJDRhcSZIaMLiSJDVgcCVJamDW4Ca5KMn9STb1rJ2T5IdJNnY/x/bc9t4kdyW5M8mrFmpwSZJGyVyOcC8Gjp5h/fyqWt39XAuQ5CDgJOA3u/t8LMlugxpWkqRRNWtwq+pG4ME5Pt7xwGeq6tGq+m/gLuDQPuaTJGlJ6Oc93DOT3N6dct6jW9sX+EHPNlu6tV+QZG2SySSTU1NTfYwhSdLiN9/gXgC8AFgNbAXO69Yzw7Y10wNU1fqqmqiqibGxsXmOIUnSaJhXcKvqvqp6rKoeBz7Bz08bbwH279l0P+De/kaUJGn0zSu4SfbpuXoCsP0TzFcDJyV5epIDgVXAV/obUZKk0bditg2SXAYcCaxMsgU4GzgyyWqmTxdvBt4GUFXfTHIF8C1gG3BGVT22MKNLkjQ6Zg1uVa2ZYfnCJ9n+g8AH+xlKkqSlxm+akiSpAYMrSVIDBleSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKmBWb9pahSNr7tm2CMM3eZzjxv2CJKkHh7hSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1MGtwk1yU5P4km3rW/jbJt5PcnuTKJLt36+NJHkmysfv5+EIOL0nSqJjLEe7FwNE7rF0PvLiqfhv4DvDentvurqrV3c/pgxlTkqTRNmtwq+pG4MEd1q6rqm3d1VuA/RZgNkmSloxBvIf7FuDfeq4fmORrSb6U5Iid3SnJ2iSTSSanpqYGMIYkSYtXX8FN8n5gG/CpbmkrcEBVHQy8E/h0kufOdN+qWl9VE1U1MTY21s8YkiQtevMObpJTgFcDb6yqAqiqR6vqge7yrcDdwAsHMagkSaNsXsFNcjTwHuA1VfVwz/pYkt26y88HVgH3DGJQSZJG2YrZNkhyGXAksDLJFuBspj+V/HTg+iQAt3SfSH458IEk24DHgNOr6sEZH1iSpGVk1uBW1ZoZli/cybYbgA39DiVJ0lLjN01JktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktTAnIKb5KIk9yfZ1LO2Z5Lrk3y3+71Ht54kf5/kriS3JzlkoYaXJGlUzPUI92Lg6B3W1gE3VNUq4IbuOsAxwKruZy1wQf9jSpI02uYU3Kq6EXhwh+XjgUu6y5cAr+1Zv7Sm3QLsnmSfQQwrSdKo6uc93L2raitA93uvbn1f4Ac9223p1p4gydokk0kmp6am+hhDkqTFbyE+NJUZ1uoXFqrWV9VEVU2MjY0twBiSJC0e/QT3vu2nirvf93frW4D9e7bbD7i3j+eRJGnk9RPcq4FTusunAFf1rL+p+7TyYcBPtp96liRpuVoxl42SXAYcCaxMsgU4GzgXuCLJacD3gdd3m18LHAvcBTwMnDrgmSVJGjlzCm5VrdnJTUfNsG0BZ/QzlCRJS43fNCVJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAZWDHsALV7j664Z9ghDt/nc44Y9gqQlwiNcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKmBeX9KOcmLgMt7lp4P/DmwO/BWYKpbf19VXTvvCSVJWgLmHdyquhNYDZBkN+CHwJXAqcD5VfWhgUwoSdISMKhTykcBd1fV9wb0eJIkLSmDCu5JwGU9189McnuSi5LsMdMdkqxNMplkcmpqaqZNJElaMvoObpKnAa8BPtstXQC8gOnTzVuB82a6X1Wtr6qJqpoYGxvrdwxJkha1QRzhHgPcVlX3AVTVfVX1WFU9DnwCOHQAzyFJ0kgbRHDX0HM6Ock+PbedAGwawHNIkjTS+vrjBUmeCfwh8Lae5b9JshooYPMOt0mStCz1Fdyqehj4lR3WTu5rIkmSliC/aUqSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAZW9PsASTYDDwGPAduqaiLJnsDlwDiwGTixqn7c73NJkjSqBnWE+/tVtbqqJrrr64AbqmoVcEN3XZKkZavvI9ydOB44srt8CfBF4D0L9FzSoja+7pphjzB0m889btgjSEM3iCPcAq5LcmuStd3a3lW1FaD7vdeOd0qyNslkksmpqakBjCFJ0uI1iCPcw6vq3iR7Adcn+fZc7lRV64H1ABMTEzWAOSRJWrT6PsKtqnu73/cDVwKHAvcl2Qeg+31/v88jSdIo6yu4SZ6V5DnbLwOvBDYBVwOndJudAlzVz/NIkjTq+j2lvDdwZZLtj/XpqvpCkq8CVyQ5Dfg+8Po+n0eSpJHWV3Cr6h7gJTOsPwAc1c9jS5K0lPhNU5IkNWBwJUlqwOBKktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNTDv4CbZP8l/JrkjyTeT/Gm3fk6SHybZ2P0cO7hxJUkaTSv6uO824F1VdVuS5wC3Jrm+u+38qvpQ/+NJkrQ0zDu4VbUV2NpdfijJHcC+gxpMkqSlZCDv4SYZBw4GvtwtnZnk9iQXJdljJ/dZm2QyyeTU1NQgxpAkadHqO7hJng1sAM6qqp8CFwAvAFYzfQR83kz3q6r1VTVRVRNjY2P9jiFJ0qLWV3CTPJXp2H6qqj4PUFX3VdVjVfU48Ang0P7HlCRptPXzKeUAFwJ3VNWHe9b36dnsBGDT/MeTJGlp6OdTyocDJwPfSLKxW3sfsCbJaqCAzcDb+ppQkqQloJ9PKd8MZIabrp3/OJIkLU1+05QkSQ0YXEmSGjC4kiQ1YHAlSWrA4EqS1IDBlSSpAYMrSVIDBleSpAYMriRJDRhcSZIaMLiSJDVgcCVJasDgSpLUgMGVJKkBgytJUgMGV5KkBgyuJEkNGFxJkhowuJIkNWBwJUlqwOBKktSAwZUkqQGDK0lSAwZXkqQGDK4kSQ0sWHCTHJ3kziR3JVm3UM8jSdIoWJDgJtkN+ChwDHAQsCbJQQvxXJIkjYKFOsI9FLirqu6pqp8BnwGOX6DnkiRp0UtVDf5Bkz8Gjq6qP+munwy8tKrO7NlmLbC2u/oi4M6BDzJcK4EfDXuIJcD92D/34WC4Hwdjqe3HX6uqsblsuGKBBsgMa08oe1WtB9Yv0PMPXZLJqpoY9hyjzv3YP/fhYLgfB2M578eFOqW8Bdi/5/p+wL0L9FySJC16CxXcrwKrkhyY5GnAScDVC/RckiQtegtySrmqtiU5E/h3YDfgoqr65kI81yK2ZE+XN+Z+7J/7cDDcj4OxbPfjgnxoSpIkPZHfNCVJUgMGV5KkBgzuk0jyxSSv2mHtrCQfm2HbSnJez/V3JzmnwZgjy33WP/fhYCV5LMnGJJuSfDbJM4c902K1i6+P7lcM7mwuY/oT1r1O6tZ39CjwuiQrF3yqpcN91j/34WA9UlWrq+rFwM+A04c90CK2K6+P7lcM7mw+B7w6ydMBkowDzwNunmHbbUx/+u7PdrwhyViSDUm+2v0c3rN+fZLbkvxjku8tsxdO91n/5rMPz0ny7p7tNnX/tvVENwG/DpDknd1+2pTkrG7tWUmuSfL1bv0NQ522vV15fey1bPerwX0SVfUA8BXg6G7pJODy2vlHuz8KvDHJL++w/hHg/Kr6XeCPgE9262cD/1FVhwBXAgcMcv4R4T7r367uQ80iyQqm//jKN5L8DnAq8FLgMOCtSQ5m+nXh3qp6SXfk9oWhDTwE83h9XPb7daG+2nEp2X7a5Kru91t2tmFV/TTJpcA7gEd6bnoFcFDy/994+dwkzwF+Dzihu+8Xkvx48OMvbu6z/s1jH2rnnpFkY3f5JuBC4O3AlVX1vwBJPg8cwXQIPpTkr4F/raqbhjHwkM319dH9isGdi38BPpzkEOAZVXXbLNv/HXAb8E89a08BXlZVvS+GpOeVcJlzn/VvV/bhNp54duuXFn68kfFIVa3uXdjZv7mq+k53lHYs8FdJrquqD7QYchGZ6+uj+xVPKc+qqv4H+CJwETN/GGDH7R8ErgBO61m+Duj9S0nb/+HdDJzYrb0S2GMgQ48Y91n/dnEfbgYO6dYOAQ5sM+XIuhF4bZJnJnkW02dYbkryPODhqvpn4EN0+3Q52dXXxx0su/1qcOfmMuAlTP9d37k4j+k/QbXdO4CJJLcn+RY//4TeXwCvTHIb0+9rbAUeGszII8d91r+57sMNwJ7dKb63A99pO+Zo6Y7aLmb6/covA5+sqq8BvwV8pduP7wf+cmhDDteuvj4Cy3O/+tWOQ9R9uu+x7runXwZcsONpFz2R+0zSqPI93OE6ALgiyVOY/r9pbx3yPKPAfSZpJHmEK0lSA76HK0lSAwZXkqQGDK4kSQ0YXEmSGjC4kiQ18H8R1QAIwI7igQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_axes([0,0,1,1])\n",
    "Sentiments = ['V Neg', 'Neg', 'Neu', 'Pos', 'V Pos']\n",
    "counts = [len(d4.Label[d4.Label == -2]),len(d4.Label[d4.Label == -1]),len(d4.Label[d4.Label == 0]),len(d4.Label[d4.Label == 1]),len(d4.Label[d4.Label == 2])]\n",
    "ax.bar(Sentiments,counts)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"L&T share price gains 3% after Credit Suisse upgrades stock to 'outperform'\""
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d4.index[d4.Label == 2]\n",
    "d4.iloc[182].title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[139, 82, 51, 203, 176]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = [len(d4.Label[d4.Label == -2]),len(d4.Label[d4.Label == -1]),len(d4.Label[d4.Label == 0]),len(d4.Label[d4.Label == 1]),len(d4.Label[d4.Label == 2])]\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "221"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(d4.Label[d4.Label < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6316666666666667"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "379/(379+221)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 1    203\n",
       " 2    176\n",
       "-2    139\n",
       "-1     82\n",
       " 0     51\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d4.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      max financial gains 5% on max life extending i...\n",
       "1      looking to invest in us market? subscription f...\n",
       "2      timing the market is not possible, but investo...\n",
       "3      zee entertainment share price plunges 14%, loo...\n",
       "4      bandhan bank share price slips 10% despite str...\n",
       "5      cadila healthcare shares rise 2% on usfda nod ...\n",
       "6      this radhakishan damani-owned stock might be d...\n",
       "7      ‘allocate 30%-40% of investable corpus now at ...\n",
       "8      indoco remedies surges 16% as it ships paracet...\n",
       "9      gpt infraprojects jumps 20% on road-widening o...\n",
       "10                    polyplex corp share price falls 5%\n",
       "11     d-street buzz: 300 stocks hit lower circuit on...\n",
       "12     slideshow | winners & losers: 10 stocks that m...\n",
       "13     d-street talk podcast | these 7 ‘phoenix' stoc...\n",
       "14     lupin share price gains 8% after nagpur unit r...\n",
       "15     ig petrochemicals share locked in upper circui...\n",
       "16     l&t share price jumps 5% on order wins for wat...\n",
       "17     gaja capital infuses rs 204 crore in sachin ba...\n",
       "18     lockdown fallout: icicidirect lowers sensex ta...\n",
       "19     aia engineering share price jumps 17% as work ...\n",
       "20     analysts suggest these 15 quality mid-caps for...\n",
       "21     caplin point climbs 7% as subsidiary gets us f...\n",
       "22     dr reddy's rises 6% on cancer drug launch, eir...\n",
       "23     the market podcast | this fund manager managin...\n",
       "24     slideshow | bajaj finance, hero moto, axis ban...\n",
       "25     cipla, icici bank, infosys among 7 stocks that...\n",
       "26     deploy nifty modified put butterfly strategy a...\n",
       "27     'after lockdown, v-shaped recovery possible in...\n",
       "28     titan q4fy20 preview: a good start to the quar...\n",
       "29     stay with equities! 3 potential signs which su...\n",
       "                             ...                        \n",
       "164    sell in may and go away? 10-year data suggests...\n",
       "165    rakesh jhunjhunwala increases stake in rallis ...\n",
       "166    d-street buzz: auto stocks tumble after april ...\n",
       "167    any correction near 9,400 level would be a goo...\n",
       "168    ril may see profit-booking on monday due to gl...\n",
       "169    time to turn slightly cautions after april ral...\n",
       "170    auto index up 30% from lows in a month; buy on...\n",
       "171    story in a chart: inverse head and shoulders p...\n",
       "172    bharti airtel share price jumps 3% after 4g de...\n",
       "173    hul to underperform in short term given expens...\n",
       "174    investor wealth plummets rs 5.15 lakh crore in...\n",
       "175    momentum plays: these 5 stocks could give 15-2...\n",
       "176    motherson sumi shares jump 9% after morgan sta...\n",
       "177    the market podcast | pankaj pandey lists out 1...\n",
       "178    glenmark pharma share price up 9% on nod to tr...\n",
       "179    ril share price rises 2% ahead of march quarte...\n",
       "180    hexaware technologies share price falls 3% pos...\n",
       "181    consumer discretionary & pharma to lead the ne...\n",
       "182    two stocks doubled while 19 rose over 50% in b...\n",
       "183    investors' can hold hul post march quarter res...\n",
       "184    'sbi card among 3 companies that can re-captur...\n",
       "185    ril to raise rs 53,000 crore via 1:15 rights i...\n",
       "186    d-street buzz: nifty auto spikes 5% led by tat...\n",
       "187    promoters raise stake in over 60 companies in ...\n",
       "188    brokerages recommend these 12 stocks in time o...\n",
       "189    morepen labs share price jumps 5% after uk sta...\n",
       "190    brokerages initiate buy on 11 stocks after cov...\n",
       "191    just dial share price jumps 7% after board app...\n",
       "192    investor wealth jumps rs 7.68 lakh crore in fo...\n",
       "193    sensex reclaims mount 33k: 4 factors that fuel...\n",
       "Name: title, Length: 651, dtype: object"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d4.title = d4.title.str.lower()\n",
    "d4.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'max financial gains 5% on max life extending insurance partnership with yes bank'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d4.iloc[0].title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stops = set(stopwords.words(\"english\")) \n",
    "from nltk.stem import PorterStemmer\n",
    "stemming = PorterStemmer()\n",
    "def preprocessor(line):\n",
    "    line = line.lower()\n",
    "    line = line.split()\n",
    "    #print(line)\n",
    "    line = [stemming.stem(i) for i in line]\n",
    "    #print(line)\n",
    "    line = [i for i in line if not i in stops]\n",
    "    #print(line)\n",
    "    return ' '.join(line)\n",
    "d4.title = [preprocessor(i) for i in d4.summary]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 651 entries, 0 to 193\n",
      "Data columns (total 9 columns):\n",
      "Label           651 non-null int64\n",
      "Unnamed: 0      238 non-null float64\n",
      "article_text    651 non-null object\n",
      "author          606 non-null object\n",
      "description     642 non-null object\n",
      "summary         651 non-null object\n",
      "time            651 non-null object\n",
      "title           651 non-null object\n",
      "url             651 non-null object\n",
      "dtypes: float64(1), int64(1), object(7)\n",
      "memory usage: 50.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df = d4\n",
    "d4.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simplifier(l):\n",
    "    if l < 0:\n",
    "        return -1\n",
    "    elif l > 0:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "df.Label = [simplifier(i) for i in df.Label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = df['summary']\n",
    "y = df['Label']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# Naïve Bayes:\n",
    "text_clf_nb = Pipeline([('tfidf', TfidfVectorizer()),\n",
    "                     ('clf', MultinomialNB()),\n",
    "])\n",
    "\n",
    "# Linear SVC:\n",
    "text_clf_lsvc = Pipeline([('tfidf', TfidfVectorizer()),\n",
    "                     ('clf', LinearSVC()),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 5  0  0 32 10]\n",
      " [ 0  0  0 26  7]\n",
      " [ 0  0  1 17  3]\n",
      " [ 0  0  0 45 15]\n",
      " [ 0  0  0 28 26]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -2       1.00      0.11      0.19        47\n",
      "          -1       0.00      0.00      0.00        33\n",
      "           0       1.00      0.05      0.09        21\n",
      "           1       0.30      0.75      0.43        60\n",
      "           2       0.43      0.48      0.45        54\n",
      "\n",
      "   micro avg       0.36      0.36      0.36       215\n",
      "   macro avg       0.55      0.28      0.23       215\n",
      "weighted avg       0.51      0.36      0.29       215\n",
      "\n",
      "0.3581395348837209\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[20  5  2 15  5]\n",
      " [10  2  1 10 10]\n",
      " [ 5  0  4  7  5]\n",
      " [ 6  1  1 30 22]\n",
      " [ 4  2  0 17 31]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -2       0.44      0.43      0.43        47\n",
      "          -1       0.20      0.06      0.09        33\n",
      "           0       0.50      0.19      0.28        21\n",
      "           1       0.38      0.50      0.43        60\n",
      "           2       0.42      0.57      0.49        54\n",
      "\n",
      "   micro avg       0.40      0.40      0.40       215\n",
      "   macro avg       0.39      0.35      0.34       215\n",
      "weighted avg       0.39      0.40      0.38       215\n",
      "\n",
      "0.4046511627906977\n"
     ]
    }
   ],
   "source": [
    "text_clf_lsvc.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_lsvc.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['3_Labels'] = [simplifier(i) for i in df.Label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_POSNEG = df[df['3_Labels'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>article_text</th>\n",
       "      <th>author</th>\n",
       "      <th>description</th>\n",
       "      <th>summary</th>\n",
       "      <th>time</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>3_Labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>max financi servic share price ralli 4.7 perce...</td>\n",
       "      <td>Sunil Matkar</td>\n",
       "      <td>Max Life-Yes Bank relationship has over the ye...</td>\n",
       "      <td>max financi servic share price ralli 4.7 perce...</td>\n",
       "      <td>April 13, 2020 12:04 PM IST</td>\n",
       "      <td>max financi gain 5% max life extend insur part...</td>\n",
       "      <td>https://www.moneycontrol.com/news/business/mar...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>zee entertain share price plung 14 percent int...</td>\n",
       "      <td>Nishant Kumar</td>\n",
       "      <td>The company will extend financial and operatio...</td>\n",
       "      <td>zee entertain share price plung 14 percent int...</td>\n",
       "      <td>April 13, 2020 02:34 PM IST</td>\n",
       "      <td>zee entertain share price plung 14%, look cour...</td>\n",
       "      <td>https://www.moneycontrol.com/news/business/mar...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>bandhan bank share price fell nearli 10 percen...</td>\n",
       "      <td>Sunil Matkar</td>\n",
       "      <td>The capital adequacy ratio of the bank at the ...</td>\n",
       "      <td>bandhan bank share price fell nearli 10 percen...</td>\n",
       "      <td>April 13, 2020 11:45 AM IST</td>\n",
       "      <td>bandhan bank share price slip 10% despit stron...</td>\n",
       "      <td>https://www.moneycontrol.com/news/business/mar...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>share price cadila healthcar wa 2 percent intr...</td>\n",
       "      <td>Sandip Das</td>\n",
       "      <td>Zydus Cadila has received tentative approval f...</td>\n",
       "      <td>share price cadila healthcar wa 2 percent intr...</td>\n",
       "      <td>April 13, 2020 12:07 PM IST</td>\n",
       "      <td>cadila healthcar share rise 2% usfda nod marke...</td>\n",
       "      <td>https://www.moneycontrol.com/news/business/sto...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>8.0</td>\n",
       "      <td>share indoco remedi ralli nearli 16 percent in...</td>\n",
       "      <td>Sunil Matkar</td>\n",
       "      <td>The permission granted by the Indian Governmen...</td>\n",
       "      <td>share indoco remedi ralli nearli 16 percent in...</td>\n",
       "      <td>April 13, 2020 01:33 PM IST</td>\n",
       "      <td>indoco remedi surg 16% ship paracetamol tablet UK</td>\n",
       "      <td>https://www.moneycontrol.com/news/business/mar...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Label  Unnamed: 0                                       article_text  \\\n",
       "0      1         0.0  max financi servic share price ralli 4.7 perce...   \n",
       "3     -2         3.0  zee entertain share price plung 14 percent int...   \n",
       "4     -1         4.0  bandhan bank share price fell nearli 10 percen...   \n",
       "5      1         5.0  share price cadila healthcar wa 2 percent intr...   \n",
       "8      2         8.0  share indoco remedi ralli nearli 16 percent in...   \n",
       "\n",
       "          author                                        description  \\\n",
       "0   Sunil Matkar  Max Life-Yes Bank relationship has over the ye...   \n",
       "3  Nishant Kumar  The company will extend financial and operatio...   \n",
       "4   Sunil Matkar  The capital adequacy ratio of the bank at the ...   \n",
       "5     Sandip Das  Zydus Cadila has received tentative approval f...   \n",
       "8   Sunil Matkar  The permission granted by the Indian Governmen...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  max financi servic share price ralli 4.7 perce...   \n",
       "3  zee entertain share price plung 14 percent int...   \n",
       "4  bandhan bank share price fell nearli 10 percen...   \n",
       "5  share price cadila healthcar wa 2 percent intr...   \n",
       "8  share indoco remedi ralli nearli 16 percent in...   \n",
       "\n",
       "                          time  \\\n",
       "0  April 13, 2020 12:04 PM IST   \n",
       "3  April 13, 2020 02:34 PM IST   \n",
       "4  April 13, 2020 11:45 AM IST   \n",
       "5  April 13, 2020 12:07 PM IST   \n",
       "8  April 13, 2020 01:33 PM IST   \n",
       "\n",
       "                                               title  \\\n",
       "0  max financi gain 5% max life extend insur part...   \n",
       "3  zee entertain share price plung 14%, look cour...   \n",
       "4  bandhan bank share price slip 10% despit stron...   \n",
       "5  cadila healthcar share rise 2% usfda nod marke...   \n",
       "8  indoco remedi surg 16% ship paracetamol tablet UK   \n",
       "\n",
       "                                                 url  3_Labels  \n",
       "0  https://www.moneycontrol.com/news/business/mar...         1  \n",
       "3  https://www.moneycontrol.com/news/business/mar...        -1  \n",
       "4  https://www.moneycontrol.com/news/business/mar...        -1  \n",
       "5  https://www.moneycontrol.com/news/business/sto...         1  \n",
       "8  https://www.moneycontrol.com/news/business/mar...         1  "
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_POSNEG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2 21]\n",
      " [ 0 37]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       1.00      0.09      0.16        23\n",
      "           1       0.64      1.00      0.78        37\n",
      "\n",
      "   micro avg       0.65      0.65      0.65        60\n",
      "   macro avg       0.82      0.54      0.47        60\n",
      "weighted avg       0.78      0.65      0.54        60\n",
      "\n",
      "0.65\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df_POSNEG['summary']\n",
    "y = df_POSNEG['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  0  0]\n",
      " [ 7  0 15  5]\n",
      " [ 1  0 21 11]\n",
      " [ 0  0  0  0]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -2       0.00      0.00      0.00         0\n",
      "          -1       0.00      0.00      0.00        27\n",
      "           1       0.58      0.64      0.61        33\n",
      "           2       0.00      0.00      0.00         0\n",
      "\n",
      "   micro avg       0.35      0.35      0.35        60\n",
      "   macro avg       0.15      0.16      0.15        60\n",
      "weighted avg       0.32      0.35      0.33        60\n",
      "\n",
      "0.35\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df_POSNEG['summary']\n",
    "y = df_POSNEG['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_lsvc.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 8 16]\n",
      " [ 2 34]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.80      0.33      0.47        24\n",
      "           1       0.68      0.94      0.79        36\n",
      "\n",
      "   micro avg       0.70      0.70      0.70        60\n",
      "   macro avg       0.74      0.64      0.63        60\n",
      "weighted avg       0.73      0.70      0.66        60\n",
      "\n",
      "0.7\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df_POSNEG['title']\n",
    "y = df_POSNEG['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[18  5]\n",
      " [ 0 37]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       1.00      0.78      0.88        23\n",
      "           1       0.88      1.00      0.94        37\n",
      "\n",
      "   micro avg       0.92      0.92      0.92        60\n",
      "   macro avg       0.94      0.89      0.91        60\n",
      "weighted avg       0.93      0.92      0.91        60\n",
      "\n",
      "0.9166666666666666\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df_POSNEG['title']\n",
    "y = df_POSNEG['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_lsvc.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1 17]\n",
      " [ 0 42]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       1.00      0.06      0.11        18\n",
      "           1       0.71      1.00      0.83        42\n",
      "\n",
      "   micro avg       0.72      0.72      0.72        60\n",
      "   macro avg       0.86      0.53      0.47        60\n",
      "weighted avg       0.80      0.72      0.61        60\n",
      "\n",
      "0.7166666666666667\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df_POSNEG['article_text']\n",
    "y = df_POSNEG['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1 23]\n",
      " [ 0 36]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       1.00      0.04      0.08        24\n",
      "           1       0.61      1.00      0.76        36\n",
      "\n",
      "   micro avg       0.62      0.62      0.62        60\n",
      "   macro avg       0.81      0.52      0.42        60\n",
      "weighted avg       0.77      0.62      0.49        60\n",
      "\n",
      "0.6166666666666667\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df_POSNEG['article_text']\n",
    "y = df_POSNEG['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_lsvc.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[10  0  9]\n",
      " [ 1  0  5]\n",
      " [ 0  0 41]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.91      0.53      0.67        19\n",
      "           0       0.00      0.00      0.00         6\n",
      "           1       0.75      1.00      0.85        41\n",
      "\n",
      "   micro avg       0.77      0.77      0.77        66\n",
      "   macro avg       0.55      0.51      0.51        66\n",
      "weighted avg       0.72      0.77      0.72        66\n",
      "\n",
      "0.7727272727272727\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df['title']\n",
    "y = df['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  0 19]\n",
      " [ 0  0  5]\n",
      " [ 0  0 41]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       1.00      0.05      0.10        20\n",
      "           0       0.00      0.00      0.00         5\n",
      "           1       0.63      1.00      0.77        41\n",
      "\n",
      "   micro avg       0.64      0.64      0.64        66\n",
      "   macro avg       0.54      0.35      0.29        66\n",
      "weighted avg       0.69      0.64      0.51        66\n",
      "\n",
      "0.6363636363636364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df['summary']\n",
    "y = df['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  0 18]\n",
      " [ 0  0  5]\n",
      " [ 0  0 42]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       1.00      0.05      0.10        19\n",
      "           0       0.00      0.00      0.00         5\n",
      "           1       0.65      1.00      0.79        42\n",
      "\n",
      "   micro avg       0.65      0.65      0.65        66\n",
      "   macro avg       0.55      0.35      0.30        66\n",
      "weighted avg       0.70      0.65      0.53        66\n",
      "\n",
      "0.6515151515151515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df['article_text']\n",
    "y = df['3_Labels']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  0 14  2]\n",
      " [ 0  0  0  6  1]\n",
      " [ 0  0  0  4  2]\n",
      " [ 0  0  0 13  7]\n",
      " [ 1  0  0  8  8]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -2       0.00      0.00      0.00        16\n",
      "          -1       0.00      0.00      0.00         7\n",
      "           0       0.00      0.00      0.00         6\n",
      "           1       0.29      0.65      0.40        20\n",
      "           2       0.40      0.47      0.43        17\n",
      "\n",
      "   micro avg       0.32      0.32      0.32        66\n",
      "   macro avg       0.14      0.22      0.17        66\n",
      "weighted avg       0.19      0.32      0.23        66\n",
      "\n",
      "0.3181818181818182\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df['article_text']\n",
    "y = df['Label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_nb.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_nb.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 4  0  0  3  3]\n",
      " [ 6  2  0  4  0]\n",
      " [ 1  0  1  1  1]\n",
      " [ 5  0  0 10  4]\n",
      " [ 1  0  0  7 13]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -2       0.24      0.40      0.30        10\n",
      "          -1       1.00      0.17      0.29        12\n",
      "           0       1.00      0.25      0.40         4\n",
      "           1       0.40      0.53      0.45        19\n",
      "           2       0.62      0.62      0.62        21\n",
      "\n",
      "   micro avg       0.45      0.45      0.45        66\n",
      "   macro avg       0.65      0.39      0.41        66\n",
      "weighted avg       0.59      0.45      0.45        66\n",
      "\n",
      "0.45454545454545453\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df['article_text']\n",
    "y = df['Label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "text_clf_lsvc.fit(X_train, y_train)\n",
    "# Form a prediction set\n",
    "predictions = text_clf_lsvc.predict(X_test)\n",
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\nltk\\twitter\\__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    }
   ],
   "source": [
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "sid = SentimentIntensityAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "E:\\Anaconda\\envs\\nlp_course\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "df_POSNEG['scores'] = df_POSNEG['article_text'].apply(lambda review: sid.polarity_scores(review))\n",
    "df_POSNEG['compound']  = df_POSNEG['scores'].apply(lambda score_dict: score_dict['compound'])\n",
    "df_POSNEG['comp_score'] = df_POSNEG['compound'].apply(lambda c: 1 if c >=0 else -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 39 182]\n",
      " [ 21 358]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.65      0.18      0.28       221\n",
      "           1       0.66      0.94      0.78       379\n",
      "\n",
      "   micro avg       0.66      0.66      0.66       600\n",
      "   macro avg       0.66      0.56      0.53       600\n",
      "weighted avg       0.66      0.66      0.59       600\n",
      "\n",
      "0.6616666666666666\n"
     ]
    }
   ],
   "source": [
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(df_POSNEG['3_Labels'],df_POSNEG['comp_score']))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(df_POSNEG['3_Labels'],df_POSNEG['comp_score']))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(df_POSNEG['3_Labels'],df_POSNEG['comp_score']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      0.6249\n",
       "1      0.0000\n",
       "2      0.0000\n",
       "3      0.8074\n",
       "4      0.7964\n",
       "5      0.2960\n",
       "6      0.0000\n",
       "7      0.0000\n",
       "8      0.0000\n",
       "9      0.0000\n",
       "10     0.2960\n",
       "11    -0.2960\n",
       "12     0.1027\n",
       "13     0.0000\n",
       "14     0.6808\n",
       "15     0.2960\n",
       "16     0.7184\n",
       "17     0.0000\n",
       "18    -0.2960\n",
       "19     0.2960\n",
       "20     0.0000\n",
       "21     0.0000\n",
       "22    -0.6597\n",
       "23     0.6249\n",
       "24     0.5574\n",
       "25     0.0000\n",
       "26     0.0000\n",
       "27     0.0000\n",
       "28     0.4404\n",
       "29     0.0000\n",
       "        ...  \n",
       "164    0.0000\n",
       "165    0.0000\n",
       "166   -0.6486\n",
       "167    0.4404\n",
       "168    0.7184\n",
       "169    0.0000\n",
       "170   -0.2732\n",
       "171    0.0000\n",
       "172    0.2960\n",
       "173    0.0000\n",
       "174    0.4939\n",
       "175    0.2500\n",
       "176   -0.0772\n",
       "177    0.4939\n",
       "178    0.2960\n",
       "179    0.2960\n",
       "180    0.2960\n",
       "181    0.0000\n",
       "182    0.0000\n",
       "183    0.0000\n",
       "184   -0.3182\n",
       "185    0.2960\n",
       "186    0.0000\n",
       "187    0.2263\n",
       "188    0.0258\n",
       "189    0.2960\n",
       "190    0.0000\n",
       "191    0.5267\n",
       "192    0.4939\n",
       "193    0.0000\n",
       "Name: compound, Length: 651, dtype: float64"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['compound']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-2, -1, 0, 1, 2]"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def vader_simplifier(l):\n",
    "    if l >= -1 and l <= -0.5:\n",
    "        return -2\n",
    "    elif l > -0.5 and l < 0:\n",
    "        return -1\n",
    "    elif l == 0:\n",
    "        return 0\n",
    "    elif l > 0 and l <= 0.5:\n",
    "        return 1\n",
    "    elif l > 0.5 and l <= 1:\n",
    "        return 2\n",
    "    \n",
    "l = [-0.89, -0.22, 0, 0.11, 0.88]\n",
    "l = [vader_simplifier(i) for i in l]\n",
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['vader_comm'] = [vader_simplifier(i) for i in df['compound']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[16 40 45 28 10]\n",
      " [ 5 17 23 31  6]\n",
      " [ 3  5 17 23  3]\n",
      " [ 5 11 77 75 35]\n",
      " [ 1 13 85 57 20]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -2       0.53      0.12      0.19       139\n",
      "          -1       0.20      0.21      0.20        82\n",
      "           0       0.07      0.33      0.11        51\n",
      "           1       0.35      0.37      0.36       203\n",
      "           2       0.27      0.11      0.16       176\n",
      "\n",
      "   micro avg       0.22      0.22      0.22       651\n",
      "   macro avg       0.28      0.23      0.21       651\n",
      "weighted avg       0.33      0.22      0.23       651\n",
      "\n",
      "0.2227342549923195\n"
     ]
    }
   ],
   "source": [
    "# Report the confusion matrix\n",
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(df['Label'],df['vader_comm']))\n",
    "# Print a classification report\n",
    "print(metrics.classification_report(df['Label'],df['vader_comm']))\n",
    "# Print the overall accuracy\n",
    "print(metrics.accuracy_score(df['Label'],df['vader_comm']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}