SPAM, HAM filter using NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nltk.download_shell()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = pd.read_csv('https://raw.githubusercontent.com/juinc/python_data_science_and_machine_learning_bootcamp/master/Machine%20Learning%20Sections/Natural-Language-Processing/smsspamcollection/SMSSpamCollection',\n",
    "                      sep = '\\t', names = ['label', 'message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">message</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>unique</th>\n",
       "      <th>top</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ham</th>\n",
       "      <td>4825</td>\n",
       "      <td>4516</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam</th>\n",
       "      <td>747</td>\n",
       "      <td>653</td>\n",
       "      <td>Please call our customer service representativ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      message                                                               \n",
       "        count unique                                                top freq\n",
       "label                                                                       \n",
       "ham      4825   4516                             Sorry, I'll call later   30\n",
       "spam      747    653  Please call our customer service representativ...    4"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.groupby('label').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "messages['length'] = messages['message'].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message  length\n",
       "0   ham  Go until jurong point, crazy.. Available only ...     111\n",
       "1   ham                      Ok lar... Joking wif u oni...      29\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...     155\n",
       "3   ham  U dun say so early hor... U c already then say...      49\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...      61"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x16a65780>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD3CAYAAADi8sSvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADvJJREFUeJzt3V2MXOddx/Hv2hvbGNbWSmwaKkVECPjLN24lV6HUuDZR\n3lwpGBAvEipQLFxHsuSAIpE35wY5xJQ0Uq2qSrXBOK3oTR1KwZJJEG6MY16cliDVqvuPjIq46M0S\n/LLBtYvt5eKc7Uw3uzOzk5ndnWe+n6szz5kz88xfu7/zzHNeZmRmZgZJUhlWLXcHJEm9Y6hLUkEM\ndUkqiKEuSQUx1CWpIKPL+eZTU9Ndn3ozPr6eixev9rI7A8taNFiLBmvRUFotJibGRhZaN7Aj9dHR\n1cvdhRXDWjRYiwZr0TBMtRjYUJckvZuhLkkFMdQlqSCGuiQVxFCXpIIY6pJUEENdkgpiqEtSQTq6\nojQi/g24Uj/8DvAMcBSYAc4B+zLzVkTsAfYCN4CDmXm85z2WJC2obahHxDpgJDN3NLX9DXAgM1+L\niBeAXRHxz8B+4EPAOuD1iPj7zLzen67Pb/ehkwAcefyepXxbSVoROhmpfwBYHxGv1s9/EtgCnKrX\nnwDuB24CZ+oQvx4RF4DNwBs977UkaV6dhPpV4DngReBnqEJ8JDNnb8Y1DWwENgCXm7abbV/Q+Pj6\n93RPhomJsa7WlWjYPm8r1qLBWjQMSy06CfW3gAt1iL8VEW9TjdRnjQGXqObcx+ZpX9B7uWvaxMQY\nU1PTC65vta407WoxTKxFg7VoKK0WrXZQnZz9shv4NEBEvJ9qRP5qROyo1+8ETgNngW0RsS4iNgKb\nqA6iLovdh07+YH5dkoZFJyP1PweORsTrVGe77Ab+G5iMiDXAeeBYZt6MiMNUAb8KeCozr/Wp35Kk\nebQN9cz8PvBb86zaPs9zJ4HJHvRLktQFLz6SpIIY6pJUEENdkgpiqEtSQQx1SSqIoS5JBTHUJakg\nhrokFcRQl6SCGOqSVBBDXZIKYqhLUkEMdUkqiKEuSQUpPtT9sQxJw6T4UJekYWKoS1JBDHVJKoih\nLkkFMdQlqSCGuiQVxFCXpIIY6pJUEENdkgpiqEtSQQx1SSqIoS5JBTHUJakghrokFcRQl6SCGOqS\nVBBDXZIKYqhLUkFGO3lSRNwOfAO4D7gBHAVmgHPAvsy8FRF7gL31+oOZebwvPZYkLajtSD0ibgM+\nD3yvbnoeOJCZ24ARYFdE3AHsB7YCDwDPRsTa/nRZkrSQTqZfngNeAL5bP94CnKqXTwD3AncDZzLz\nemZeBi4Am3vcV0lSGy2nXyLiE8BUZr4SEU/UzSOZOVMvTwMbgQ3A5aZNZ9tbGh9fz+jo6kV3etbE\nxFhfnjuISv98i2EtGqxFw7DUot2c+m5gJiLuBT4IfAG4vWn9GHAJuFIvz21v6eLFq4vqbLOJiTGm\npqY7fv5injtoFluLklmLBmvRUFotWu2gWoZ6Zn50djkiXgMeBv4sInZk5mvATuBrwFngmYhYB6wF\nNlEdRJUkLaGOzn6Z41FgMiLWAOeBY5l5MyIOA6ep5umfysxrPeynJKkDHYd6Zu5oerh9nvWTwGQP\n+iRJ6pIXH0lSQQx1SSqIoS5JBTHUJakghrokFcRQl6SCGOqSVJChCfXdh06y+9DJ5e6GJPXV0IS6\nJA0DQ12SCmKoS1JBDHVJKoihLkkFMdQlqSCGuiQVxFCXpIIY6pJUkG5+zm6gNV9VeuTxe5axJ5LU\ne47UJakghrokFcRQl6SCGOqSVBBDXZIKYqhLUkEMdUkqiKG+AvirTJJ6xVCXpIIY6pJUEENdkgpi\nqEtSQQx1SSqIoS5JBTHUJakgbe+nHhGrgUkggBngYeAacLR+fA7Yl5m3ImIPsBe4ARzMzON96rck\naR6djNQfAsjMrcAB4BngeeBAZm4DRoBdEXEHsB/YCjwAPBsRa/vSa0nSvNqGemb+NfDJ+uFPApeA\nLcCpuu0EcC9wN3AmM69n5mXgArC55z2WJC2oo5+zy8wbEfES8CvArwH3ZeZMvXoa2AhsAC43bTbb\nvqDx8fWMjq5edKdnTUyMdb1tL7bvtffSn5X2WZaTtWiwFg3DUouOf6M0M383Ih4D/hX4kaZVY1Sj\n9yv18tz2BV28eLXzns4xMTHG1NR019sD73n7Xuu2P72oRSmsRYO1aCitFq12UG2nXyLityPiifrh\nVeAW8PWI2FG37QROA2eBbRGxLiI2ApuoDqJKkpZIJyP1vwL+IiL+EbgN+APgPDAZEWvq5WOZeTMi\nDlMF/Crgqcy81qd+S5Lm0TbUM/N/gd+YZ9X2eZ47SXX6oyRpGXjxkSQVZKhD3R+nkFSaoQ51SSqN\noS5JBTHUJakghrokFcRQl6SCGOqSVBBDXZIKYqhLUkEMdUkqiKEuSQUx1CWpIIa6JBXEUJekghjq\nklQQQ12SCmKoS1JBDHVJKoihLkkFMdQlqSCGuiQVxFCXpIIY6pJUEENdkgpiqEtSQQx1SSqIoS5J\nBTHUJakghrokFcRQl6SCGOrz2H3oJLsPnVzubkjSohnqklSQ0VYrI+I24AhwF7AWOAh8CzgKzADn\ngH2ZeSsi9gB7gRvAwcw83r9uS5Lm026k/nHg7czcBjwIfBZ4HjhQt40AuyLiDmA/sBV4AHg2Itb2\nr9uSpPm0HKkDXwaO1csjVKPwLcCpuu0EcD9wEziTmdeB6xFxAdgMvNHzHveB8+eSStEy1DPzHYCI\nGKMK9wPAc5k5Uz9lGtgIbAAuN206297S+Ph6RkdXd9HtysTEWNfbroTX7+X7LXVfVzJr0WAtGoal\nFu1G6kTEncBXgM9l5pci4lNNq8eAS8CVenlue0sXL15dXG+bTEyMMTU13fX2nej368/9htDt+y1F\nLQaFtWiwFg2l1aLVDqrlnHpEvA94FXgsM4/UzW9GxI56eSdwGjgLbIuIdRGxEdhEdRBVkrSE2o3U\nnwTGgacj4um67RHgcESsAc4DxzLzZkQcpgr4VcBTmXmtX52WJM2v3Zz6I1QhPtf2eZ47CUz2qF+S\npC548ZEkFcRQl6SCGOqSVBBDXZIKYqhLUkEMdUkqiKEuSQUx1CWpIIa6JBXEUJekghjqHfA3SyUN\nCkNdkgpiqEtSQdr+SMYwc8pF0qAx1JeBOwtJ/eL0iyQVxFCXpIIY6pJUEENdkgpiqK8gXuQk6b0y\n1CWpIIa6JBXEUJekghjqklQQQ12SCmKoL4Jnp0ha6bz3yxJyhyCp3wz1JWCYS1oqTr9IUkEMdUkq\niKHeQx5IlbTcDHVJKoihLkkF6ejsl4j4OeBPM3NHRPw0cBSYAc4B+zLzVkTsAfYCN4CDmXm8T32W\nJC2g7Ug9Iv4IeBFYVzc9DxzIzG3ACLArIu4A9gNbgQeAZyNibX+6vPzazZ07ty5puXQyUv8P4FeB\nL9aPtwCn6uUTwP3ATeBMZl4HrkfEBWAz8EZvuztYDHZJS61tqGfmyxFxV1PTSGbO1MvTwEZgA3C5\n6Tmz7S2Nj69ndHR1572dY2JirOtte2H2/R969Kt9ed1+b1Mqa9FgLRqGpRbdXFF6q2l5DLgEXKmX\n57a3dPHi1S7evjIxMcbU1HTX2/dCr8N81mI/10qoxUphLRqsRUNptWi1g+rm7Jc3I2JHvbwTOA2c\nBbZFxLqI2AhsojqIKklaQt2M1B8FJiNiDXAeOJaZNyPiMFXArwKeysxrPeynJKkDHYV6Zv4n8OF6\n+S1g+zzPmQQme9k5SdLiePGRJBXEUJekghjqklQQQ12SCmKoS1JBDHVJKoihLkkFKeaHp715liQ5\nUpekohjqklQQQ12SCmKoS1JBDHVJKoihLkkFMdQlqSCGuiQVxFCXpIIM7BWl/frRZ0kaZI7UJakg\nhrokFcRQl6SCGOqSVBBDfQXafeiktxKW1BVDXZIKYqhLUkEMdUkqiKEuSQUx1AeAB04ldWpgbxMw\nDAxySYvlSF2SCmKoS1JBDHVJKoihPoDmHjh96NGvOv8uCejxgdKIWAV8DvgAcB34/cy80Mv3GGZz\ng7tdkC+0/sjj9/SsT5JWll6f/fLLwLrM/PmI+DDwaWBXj99DC5gN8XahPTfsOwn5djuQfrznUuq0\ndtJK1+tQ/wXg7wAy818i4kM9fn11YLFTMb2Yuun2PWdDdKHH7SwUwgvtRLrZ8bV7j+XeEbTrx0Lr\nV0r/+2Elf7Z+921kZmamZy8WES8CL2fmifrxfwE/lZk3evYmkqQF9fpA6RVgrPn1DXRJWjq9DvUz\nwMcA6jn1b/b49SVJLfR6Tv0rwH0R8U/ACPB7PX59SVILPZ1TlyQtLy8+kqSCGOqSVBBDXZIKMlD3\nUx/W2xBExG3AEeAuYC1wEPgWcBSYAc4B+zLzVkTsAfYCN4CDmXl8OfrcTxFxO/AN4D6qz3mUIawD\nQEQ8AfwSsIbqf+MUQ1iP+n/kJar/kZvAHob0b2PQRuo/uA0B8DjVbQiGwceBtzNzG/Ag8FngeeBA\n3TYC7IqIO4D9wFbgAeDZiFi7TH3ui/qf9/PA9+qmoawDQETsAD5C9Tm3A3cyvPX4GDCamR8B/hh4\nhiGtxaCF+g/dhgAYltsQfBl4ul4eoRphbKEalQGcAO4F7gbOZOb1zLwMXAA2L3Ff++054AXgu/Xj\nYa0DVKH0TapTif8WOM7w1uMtYLT+Nr8B+D+GtBaDFuobgMtNj29GxEBNIXUjM9/JzOmIGAOOAQeA\nkcycPR91GtjIu+sz216EiPgEMJWZrzQ1D10dmvw41cDm14GHgb+kuop7GOvxDtXUy7eBSeAwQ/q3\nMWihPrS3IYiIO4GvAV/MzC8Bt5pWjwGXeHd9ZttLsZvq4rbXgA8CXwBub1o/LHWY9TbwSmZ+PzMT\nuMYPB9Qw1eMPqWrxs1TH3F6iOs4wa2hqMWihPpS3IYiI9wGvAo9l5pG6+c16ThVgJ3AaOAtsi4h1\nEbER2ER1gKgImfnRzNyemTuAfwd+BzgxbHVo8jrwYESMRMT7gR8F/mFI63GRxgj8f4DbGML/ERiw\nK0qbzn7ZTH0bgsz89vL2qv8i4jPAb1J9tZz1CNVXzDXAeWBPZt6sj+x/kmqH/SeZ+fJS93cp1KP1\nh6m+sUwyvHX4FPCLVJ/zSeA7DGE9IuLHqM4Q+wmqz/4Z4OsMYS0GKtQlSa0N2vSLJKkFQ12SCmKo\nS1JBDHVJKoihLkkFMdQlqSCGuiQV5P8BuH9y1gIm6mUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x16a73c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "messages['length'].hist(bins = 150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"For me the love should start with attraction.i should feel that I need her every time around me.she should be the first thing which comes in my thoughts.I would start the day and end it with her.she should be there every time I dream.love will be then when my every breath has her name.my life should happen around her.my life will be named to her.I would cry for her.will give all my happiness and take all her sorrows.I will be ready to fight with anyone for her.I will be in love when I will be doing the craziest things for her.love will be when I don't have to proove anyone that my girl is the most beautiful lady on the whole planet.I will always be singing praises for her.love will be when I start up making chicken curry and end up makiing sambar.life will be the most beautiful then.will get every morning and thank god for the day because she is with me.I would like to say a lot..will tell later..\""
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[messages['length']==910]['message'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([<matplotlib.axes._subplots.AxesSubplot object at 0x000000001997E208>,\n",
       "       <matplotlib.axes._subplots.AxesSubplot object at 0x00000000198FE9B0>], dtype=object)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt8AAAEMCAYAAAAcSNrYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG15JREFUeJzt3X+U3XV95/FnkiFJ0Uk6rldpz7pFj/VdWpvahQILhsyy\naDZIzdZztHv22DalTVlKxXI8RTB4unVhwa6ELWvRdmiKdre1K5YV0oNYDXAC24pV6MIpvilU1z3q\n2jmaH4MxgZDZP743ZRyGYebOvZ/vfL/3+TgnJ9/53u/33vfnm5v5vu7nfr6f74rp6WkkSZIkDd7K\nuguQJEmShoXhW5IkSSrE8C1JkiQVYviWJEmSCjF8S5IkSYUYviVJkqRCDN9qtIgYj4hH6q5DkiRp\nIQzfkiRJUiEjdRcg9cGLI+JjwI8Aa4HtwDeB3wVeDPwg8BDws5l5OCIOAzcAFwDrgN8A3gr8OPB1\n4Kcz8zvFWyFJ6klEvBj4Q+CHgWPAF4A/Ad4PfA14FfBdYFtmPhoRr8FzhGpiz7fa4J8CN2Tm64Df\nA/4DVQD/SGb+C+DVwCuBN3W3XwN8IzN/HLgJuBn4deBHgfXA1qLVS5KW6meA0e554Ke6614F/HPg\n+szcQBXO/6j7mOcI1cbwrTZ4IjM/111+CHgZ8G5gMiIuBz5E1bPx4hn7fOL4vsDDmfm1zDwGfBl4\nSZmyJUl9ch/wYxFxD3AF8F+Ax4G/ycy93W12AT8ZEf8EzxGqkcNO1AZPz1ieBlZQfd04AvwP4M+B\nf9Zdf9yR59lfktQwmfnliHg1MA6cC3wGeAdwdMZmK7p/nsFzhGpkz7faajPwvsz8U6pAfgawqt6S\nJEmDEBEXUw0r+XRmvhu4C/g14HURsaG72a8A92fmfjxHqEb2fKut3gPcFhHfBg4B91KN65Mktc9H\nqXq9/zYivgN8FfgdqnHf10TEycA/AD/X3d5zhGqzYnp6uu4aJEmS+ioixoEPZuZr665FmslhJ5Ik\nSVIh9nxLkiRJhdjzLUmSJBXiBZeSpL6KiDOA92fmeES8DvivVNO7HQF+PjO/GRHbgYuopoK7OjN3\n11exJJVjz7ckqW+6Ny25GVjbXfU7wDsycxz4M+DdEXEScClwNtWUb9dGxJoaypWk4pZlz/fk5NSi\nB6KPjZ3Ivn2HBlHOsmR72832NkenM7rihbcaKk8Ab+HZ23j/28z8Rnd5BDgMnE413/IR4EhEPA5s\nAD7/fE/ay3mhDZr8f6Nfhv0YDHv7oZnHYL5zw7IM370YGRmuufFtb7vZXjVVZn6iO6fy8Z+/ARAR\nZ1Hd9OQcqt7uAzN2mwLWz/e8Y2MnDu37pNMZrbuE2g37MRj29kO7jkFrwrckaXmKiJ8FdgBvyszJ\niDgIzDyTjgL753uOpvV69UunM8rk5FTdZdRq2I/BsLcfmnkM5vuwYPiWJA1MRLyd6sLK8cz8dnf1\nA1R3HVwLrAFOAR6pqURJKsrwLUkaiIhYBdxIdavvP4sIgHsz8zcj4kZgL9WF/zsy83B9lUpSOYZv\nSVJfZeZXgDO7P77kebaZACZK1SRJy4VTDUqSJEmFGL4lSZKkQgzfkiRJUiGGb0mSJKmQVl9weeF1\ne77n511XnFtTJZIkSf0xO9+AGadJ7PmWJEmSCjF8S5IkSYUYviVJkqRCFjTmOyLOAN6fmeMR8Wrg\nFmCa6nbAl2TmsYjYTnUL4aPA1Zm5OyK+D/hvwMuAKeAXMnNyAO2QJEmSlr0X7PmOiMuBm4G13VU7\ngasycyOwAtgaEScBlwJnA5uBayNiDXAx8HB3248CV/W/CZIkSVIzLGTYyRPAW2b8fCpwb3f5TuA8\n4HTg/sw8kpkHgMeBDcDrgU/N2laSJEkaSi847CQzPxERJ89YtSIzp7vLU8B6YB1wYMY2c60/vu4F\njY2dyMjIqoVs+j06ndF5H59rap47rt+66NdZLl6ovW1je9tt2NorSRpOvczzfWzG8iiwHzjYXZ5v\n/fF1L2jfvkOLLqrTGWVycmrR+/Wyz3LQa3ubyva2W5Pb64cGSdJi9BK+H4yI8cy8B9gC3A08AFwT\nEWuBNcApVBdj3g+c3318C7C3H0VLkiTpWd5YsDl6mWrwXcBvRcRfAquBWzPz/wE3UoXrPcCOzDwM\nfAj4sYi4D/gV4Lf6U7YkSZLUPAvq+c7MrwBndpcfAzbNsc0EMDFr3SHgrUuuUpIkSWoBb7IjSZIk\nFWL4liRJkgoxfEuSJEmFGL4lSZKkQgzfkiRJUiGGb0mSJKkQw7ckSZJUiOFbkiRJKsTwLUmSJBVi\n+JYkSZIKMXxLkiRJhRi+JUmSpEJG6i5AktQuEXEG8P7MHI+IVwO3ANPAI8AlmXksIrYDFwFHgasz\nc3dtBUtSQfZ8S5L6JiIuB24G1nZX7QSuysyNwApga0ScBFwKnA1sBq6NiDV11CtJpRm+JUn99ATw\nlhk/nwrc212+EzgPOB24PzOPZOYB4HFgQ9EqJakmDjuRJPVNZn4iIk6esWpFZk53l6eA9cA64MCM\nbY6vf15jYycyMrKqn6U2RqczWncJtRv2Y9BL+9t2zNrUHsO3JGmQjs1YHgX2Awe7y7PXP699+w71\nv7IG6HRGmZycqruMWg37Mei1/W06Zk18D8z3YcFhJ5KkQXowIsa7y1uAvcADwMaIWBsR64FTqC7G\nlKTWs+dbkjRI7wImImI18Chwa2Y+ExE3UgXxlcCOzDxcZ5GSVIrhW5LUV5n5FeDM7vJjwKY5tpkA\nJspWJkn1c9iJJEmSVIjhW5IkSSrE8C1JkiQVYviWJEmSCjF8S5IkSYUYviVJkqRCDN+SJElSIYZv\nSZIkqRDDtyRJklSI4VuSJEkqxPAtSZIkFWL4liRJkgoZ6WWniDgB+AhwMvAMsB04CtwCTAOPAJdk\n5rGI2A5c1H386szcvfSyJUmSpObptef7fGAkM88C3gdcA+wErsrMjcAKYGtEnARcCpwNbAaujYg1\nSy9bkiRJap5ew/djwEhErATWAU8DpwL3dh+/EzgPOB24PzOPZOYB4HFgw9JKliRJkpqpp2EnwJNU\nQ06+BLwUuAA4JzOnu49PAeupgvmBGfsdXz+vsbETGRlZteiiOp3RIvssF02uvRe2t92Grb2SpOHU\na/i+DLgrM6+MiFcAe4DVMx4fBfYDB7vLs9fPa9++Q4suqNMZZXJyatH79bLPctBre5vK9rZbk9vr\nhwZJ0mL0OuxkH8/2aH8bOAF4MCLGu+u2AHuBB4CNEbE2ItYDp1BdjClJkiQNnV57vm8AdkXEXqoe\n7/cAfw1MRMRq4FHg1sx8JiJupAriK4EdmXm4D3VLkiRJjdNT+M7MJ4G3zfHQpjm2nQAmenkdSZIk\nqU28yY4kSZJUiOFbkiRJKsTwLUmSJBVi+JYkSZIKMXxLkiRJhRi+JUmSpEIM35IkSVIhhm9JkiSp\nkF7vcClJ0oJExAnAR4CTgWeA7cBR4BZgGngEuCQzj9VUoiQVY8+3JGnQzgdGMvMs4H3ANcBO4KrM\n3AisALbWWJ8kFWP4liQN2mPASESsBNYBTwOnAvd2H78TOK+m2iSpKIedSJIG7UmqISdfAl4KXACc\nk5nT3cengPXzPcHY2ImMjKwaZI3LVqczWncJtRv2Y9BL+9t2zNrUHsO3JGnQLgPuyswrI+IVwB5g\n9YzHR4H98z3Bvn2HBlje8tXpjDI5OVV3GbUa9mPQa/vbdMya+B6Y78OCw04kSYO2DzjQXf42cALw\nYESMd9dtAfbWUJckFWfPtyRp0G4AdkXEXqoe7/cAfw1MRMRq4FHg1hrrk6RiDN+SpIHKzCeBt83x\n0KbStUhS3Rx2IkmSJBVi+JYkSZIKMXxLkiRJhRi+JUmSpEIM35IkSVIhhm9JkiSpEMO3JEmSVIjh\nW5IkSSrE8C1JkiQVYviWJEmSCjF8S5IkSYUYviVJkqRCRuouoG4XXrfnOet2XXFuDZVIkiSp7ez5\nliRJkgoxfEuSJEmF9DzsJCKuBN4MrAZuAu4FbgGmgUeASzLzWERsBy4CjgJXZ+bupRYtSZIkNVFP\nPd8RMQ6cBZwNbAJeAewErsrMjcAKYGtEnARc2t1uM3BtRKzpQ92SJElS4/Q67GQz8DBwG3AHsBs4\nlar3G+BO4DzgdOD+zDySmQeAx4ENS6pYkiRJaqheh528FPgh4ALglcDtwMrMnO4+PgWsB9YBB2bs\nd3y9JEmSNHR6Dd/fAr6UmU8BGRGHqYaeHDcK7AcOdpdnr5/X2NiJjIysWnRRnc7oC29U8HkGrSl1\n9ovtbbdha68kaTj1Gr7vA94ZETuBHwBeBHw2IsYz8x5gC3A38ABwTUSsBdYAp1BdjDmvffsOLbqg\nTmeUycmpRe83l349zyD1s71NYHvbrcnt9UODJGkxegrfmbk7Is6hCtcrgUuALwMTEbEaeBS4NTOf\niYgbgb3d7XZk5uH+lC5JkiQ1S89TDWbm5XOs3jTHdhPARK+vI0mSJLWFN9mRJEmSCjF8S5IkSYX0\nPOxEkiRJ/XXhdXues27XFefWUIkGxZ5vSZIkqRB7viVJAxcRVwJvBlYDN1HdEfkWYJpqCtpLMvNY\nbQVKUiH2fEuSBioixoGzgLOpZsV6BbATuCozNwIrgK21FShJBRm+JUmDthl4GLgNuAPYDZxK1fsN\ncCdwXj2lSVJZDjuRJA3aS4EfAi4AXgncDqzMzOnu41PA+ppqk6SiDN+SpEH7FvClzHwKyIg4TDX0\n5LhRYP98TzA2diIjI6sGWOLy1emM1l1C7Yb9GPTS/rYdsza1x/AtSRq0+4B3RsRO4AeAFwGfjYjx\nzLwH2ALcPd8T7Nt3aOBFLkedziiTk1N1l1ErjwE9tb9Nx6yJ74H5PiwYviVJA5WZuyPiHOABqmuN\nLgG+DExExGrgUeDWGkuUpGIM35KkgcvMy+dYval4IZJUM2c7kSRJkgoxfEuSJEmFGL4lSZKkQgzf\nkiRJUiGGb0mSJKkQZzuRJElaxi68bk9f9tl1xbn9KEdLZM+3JEmSVIjhW5IkSSrE8C1JkiQV4pjv\nOcweJ+UYKUmSJPWDPd+SJElSIYZvSZIkqRDDtyRJklSI4VuSJEkqxPAtSZIkFWL4liRJkgoxfEuS\nJEmFGL4lSZKkQgzfkiRJUiGGb0mSJKmQJd1ePiJeBnwBeANwFLgFmAYeAS7JzGMRsR24qPv41Zm5\ne0kVS5IkSQ3Vc893RJwA/B7w3e6qncBVmbkRWAFsjYiTgEuBs4HNwLURsWZpJUuSJEnNtJRhJx8A\nPgx8vfvzqcC93eU7gfOA04H7M/NIZh4AHgc2LOE1JUmSpMbqadhJRGwDJjPzroi4srt6RWZOd5en\ngPXAOuDAjF2Pr5/X2NiJjIysWnRdnc7oovep83mXarnWNSi2t92Grb2SpOHU65jvC4HpiDgPeB3w\nUeBlMx4fBfYDB7vLs9fPa9++Q4suqNMZZXJyatH7LcSgnncpBtne5cj2tluT2+uHBknSYvQUvjPz\nnOPLEXEP8O+B/xwR45l5D7AFuBt4ALgmItYCa4BTqC7GlCRJkobOkmY7meVdwERErAYeBW7NzGci\n4kZgL9X48h2ZebiPrylJkiQ1xpLDd2aOz/hx0xyPTwATS30dSZIkqen62fMtSdKcFnJfiPqqk6Ry\nvMOlJGmgFnJfiLpqk6TSDN+SpEFbyH0hJGkoOOxEkjQwi7gvxLx6vf9DGzidpcegX5p8HJtc+2yG\nb0nSIC30vhDz6uX+D23Q5Dnw+8Vj0D9NPY5NfA/M92HB8C1JGphF3BdCar0Lr9vznHW7rji3hkpU\nJ8O3JKm059wXouZ6JKkYw/cC+ElVkpbuhe4LIQ2juTKG2s3ZTiRJkqRC7PmWJElaotk92H5Drudj\nz7ckSZJUiOFbkiRJKsTwLUmSJBVi+JYkSZIKMXxLkiRJhRi+JUmSpEIM35IkSVIhhm9JkiSpEMO3\nJEmSVIjhW5IkSSrE8C1JkiQVYviWJEmSCjF8S5IkSYWM1F2AJElS21x43Z66S9AyZc+3JEmSVIjh\nW5IkSSrE8C1JkiQVYviWJEmSCjF8S5IkSYUYviVJkqRCDN+SJElSIc7z3aPZ83fuuuLcmiqRJElS\nU/QUviPiBGAXcDKwBrga+FvgFmAaeAS4JDOPRcR24CLgKHB1Zu5eetnLz1yT6RvIJUmSNFOvPd9v\nB76VmT8XES8BHur+uSoz74mIDwNbI+IvgUuB04C1wH0R8ReZeaQfxTeNAV2SJGm49Rq+Pw7c2l1e\nQdWrfSpwb3fdncAbgWeA+7th+0hEPA5sAD7fc8WSJElSQ/UUvjPzSYCIGKUK4VcBH8jM6e4mU8B6\nYB1wYMaux9fPa2zsREZGVi26rk5ndNH7DNJC6pndG37H9Vv7+vxtYnvbbdjaK0kaTj1fcBkRrwBu\nA27KzD+OiN+e8fAosB842F2evX5e+/YdWnQ9nc4ok5NTi95vkHqpZ6H7LMf2DpLtbbcmt9cPDfNb\nzDVCNZUoSUX1esHly4FPA7+WmZ/trn4wIsYz8x5gC3A38ABwTUSspfqlewrVL9qhMNcYb0kaMgu6\nRoiqM0eSWq/Xnu/3AGPAeyPivd117wRujIjVwKPArZn5TETcCOylmlN8R2YeXmrRkqTGWOg1QoZv\nSUOh1zHf76QK27NtmmPbCWCil9eRJDXbIq4Rmlev1wK1gUObPAb90uTj2OTaZ/MmO5KkgVrgNULz\n6uVaoDZo8vUQ/eIx6J+mHscmvgfm+7Dg7eUlSQMz4xqhd2fmru7qByNivLu8hWpooiQNBXu+JUmD\ntKBrhOoqTpJKM3xLkgZmMdcISXWYPTPZQu487WxmWgqHnUiSJEmFGL4lSZKkQgzfkiRJUiGGb0mS\nJKkQL7iUJEmahxdYqp/s+ZYkSZIKaU3P90+/65N1lyBJkiTNy55vSZIkqRDDtyRJklRIa4adSJKk\n4bDQCyAXcrdKqTR7viVJkqRCDN+SJElSIYZvSZIkqRDHfEuSpGXNm9yoTez5liRJkgqx51uSJLXS\n7B5zZz/RcmDPtyRJklSIPd+SJA0xe4elsgzfkiRJXW2+uNMPWsuDw04kSZKkQuz5liRJ/2iunt+5\nekj71Ytasje2zb3aag57viVJkqRC7PmWJKlGC+mNbcLY3H61w95ptZ3he5lZ6Nd9kiRJah7DtyRJ\nLWUvsubTlm9dmsYx35IkSVIh9nxLktQD50zWsHKI7NIYvhvAr4UkqZnaMuyjLe3Q4vlv338DD98R\nsRK4CfgJ4Ajwy5n5+KBfV731yvhpVlIJnhskDasV09PTA32BiHgL8ObM3BYRZwJXZubW+faZnJxa\ndFF+MuvN7GDday/7oEN7pzPK5ORU355vubO9zdHpjK6ou4YmWuy5oZfzAvT3d9py08vvb6mUft50\nqeQ5ol/DyeY7N5QYdvJ64FMAmflXEXFagdfUAvXyy7qfv+AHNWbSHnxp2fPcIGkolej5vhn4RGbe\n2f35q8CrMvPoQF9YkrRseW6QNKxKTDV4EBid+Zr+cpWkoee5QdJQKhG+7wfOB+iO63u4wGtKkpY3\nzw2ShlKJMd+3AW+IiP8FrAB+scBrSpKWN88NkobSwMd8S5IkSap4e3lJkiSpEMO3JEmSVIjhW5Ik\nSSqk8eG7e4tiSZIkadlr5AWXEfEqYCdwGnCU6kPEw8BlmflYnbUNSkScAGwA1gP7gUcy86l6qxoc\n22t7pWEUEVuB83j2/8Ze4NbMbN7JWupR288RTQ3fe4ArM/NzM9adCVyfmWfXV9lgRMSbgGuBvwOe\npLoxxY8A78nM/1lnbYNge22vNIwi4nepOpPuBKao/m9sAU7IzF+us7aS2h68FmKYj8EwnCNKzPM9\nCGtnBm+AzPyriKirnkHbAbw+Mw8eXxER64HPAK14I85ie22vNIxem5mbZq27PSLur6WaGjxf8IqI\n1gSvF+IxaP85oqnh+28iYhfwKeAA1RvzfOB/11rV4JwAHJq17rtA8762WBjba3ulYbQyIjZm5t7j\nKyJiE/B0jTWV1vrgtQDDfgxaf45oavj+VeDfAK8H1gEHgd1Ud0xro98HvhgR91F92FhH1fYba61q\ncGyv7ZWG0TZgZ0T8MdVdP18OfBoYmiEnDEHwWoBhPwatP0c0csz3MIqIlwOnU/XyHwQ+n5nfrLeq\nwbG9tlcaNhHxB5n5SxFxBvDfgW9RBY9ts4datlVEbAfeATwneGXmH9RZWykeg/afI5ymrznOBDYD\n/xp4I3BORKyot6SBsr22Vxo2r+z+fQ2wJTPPAP4V8Nv1lVRWZk4Ab6C66PTh7t9vHJbQCc85Bo8w\nhMeAlp8jmjrsZKjMcwX8Zlr4daTttb3SkHsmM/8OIDO/PoT3sziTKnyuo5rp4/siYmimW4yIt2bm\nxyPibuA3gdcBX4iIqzPzyZrLG7hhOEcYvpth2K6At722VxpG6yPiC8CLIuKXqIaeXA/8n3rLKmcY\ngtcCXAx8HLgB+HvgUqpvQH4f+Hc11lVK688Rw/ZpuqlWRsTGmSsi4hzaewW87W33DAfD1l5pQTLz\nVOAs4OeBzwHHqIZe/GKddRX22sy8ODNvz8y7u39fDJxSd2E1eE1mXpuZj2bmB4GT6i6okNZnAHu+\nm2Eb33sF/DHgQaoLMtpoG8+2dyXQoeoF2V5nUQO0je9t73rgs7S3l2cb3/t+Xk31fm5re6UFy8wj\nwAMzVn24rlpqMtd0i60KXgvwmoi4DHg6In4yMx+MiNOoflcOg220PPMYvpvhR6nGfD0F7MjMj8E/\n3unz3DoLG5BVwG9Q/acD+Oisn9vmHOCLwPuovmaepPo3Pxl4vL6yBmYV1Yn0Pqqpoz4KvAY4lXa2\nV9LCbaMKXn/Cs8Hri7S382UuF1D9PnwM2BARfw98kGo4yjBofeYxfDfDDuAnqELLxyNiTWZ+hPaG\n0c9QzXH6dao2/jDP9v604j/eLL8KjAO3A2/OzMci4geBT1Idi7aZAP4jVQ//HVTv7f1Ubf3TGuuS\nVLPMfALYWncddcrMh4CHgJmzm5xZUzl1aH3mMXw3w1OZuR8gIrYCeyLiq7R3wv3TqML2hzLzLyLi\n7sxsY+g+7unM/E5ETFFdXHN8hoO2/vuOZOZnutNG/afM/BpARAzT18qS5tCd4WPNXI9l5lmFy6mF\nx6D9mcfw3QxfiYidwHszcyoi3gLcBXx/zXUNRGb+Q0S8DfhARPxU3fUUcHtEfJJqPtfdEXEX1dym\ne+ota2C+EhEfo/r982REXEN1I4lv1FuWpGXgCqpvx34GOFpzLXUZ9mPQ+sxj+G6GC4G30/3Ul5n/\nNyL+JXBlrVUNUGYeBX49IrbR8ll5MvO67mwfm4GvAi+jupPZn9db2cD8AnA+1XjGJ4HLqIYZXVhn\nUZLql5mfi4g/AjZk5m1111MHj0H7M4+3l5ckSZIKaXWPoiRJkrScGL4lSZKkQgzfkiRJUiGGb0mS\nJKkQw7ckSZJUyP8HyBvXZlpwXUwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x198d6e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "messages.hist(column='length', by = 'label', bins  = 60, \n",
    "              figsize=(12,4))\n",
    "#Here we see that the Ham message is centered around 1-100\n",
    "#But spam message is certered around 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'i',\n",
       " u'me',\n",
       " u'my',\n",
       " u'myself',\n",
       " u'we',\n",
       " u'our',\n",
       " u'ours',\n",
       " u'ourselves',\n",
       " u'you',\n",
       " u\"you're\",\n",
       " u\"you've\",\n",
       " u\"you'll\",\n",
       " u\"you'd\",\n",
       " u'your',\n",
       " u'yours',\n",
       " u'yourself',\n",
       " u'yourselves',\n",
       " u'he',\n",
       " u'him',\n",
       " u'his',\n",
       " u'himself',\n",
       " u'she',\n",
       " u\"she's\",\n",
       " u'her',\n",
       " u'hers',\n",
       " u'herself',\n",
       " u'it',\n",
       " u\"it's\",\n",
       " u'its',\n",
       " u'itself',\n",
       " u'they',\n",
       " u'them',\n",
       " u'their',\n",
       " u'theirs',\n",
       " u'themselves',\n",
       " u'what',\n",
       " u'which',\n",
       " u'who',\n",
       " u'whom',\n",
       " u'this',\n",
       " u'that',\n",
       " u\"that'll\",\n",
       " u'these',\n",
       " u'those',\n",
       " u'am',\n",
       " u'is',\n",
       " u'are',\n",
       " u'was',\n",
       " u'were',\n",
       " u'be',\n",
       " u'been',\n",
       " u'being',\n",
       " u'have',\n",
       " u'has',\n",
       " u'had',\n",
       " u'having',\n",
       " u'do',\n",
       " u'does',\n",
       " u'did',\n",
       " u'doing',\n",
       " u'a',\n",
       " u'an',\n",
       " u'the',\n",
       " u'and',\n",
       " u'but',\n",
       " u'if',\n",
       " u'or',\n",
       " u'because',\n",
       " u'as',\n",
       " u'until',\n",
       " u'while',\n",
       " u'of',\n",
       " u'at',\n",
       " u'by',\n",
       " u'for',\n",
       " u'with',\n",
       " u'about',\n",
       " u'against',\n",
       " u'between',\n",
       " u'into',\n",
       " u'through',\n",
       " u'during',\n",
       " u'before',\n",
       " u'after',\n",
       " u'above',\n",
       " u'below',\n",
       " u'to',\n",
       " u'from',\n",
       " u'up',\n",
       " u'down',\n",
       " u'in',\n",
       " u'out',\n",
       " u'on',\n",
       " u'off',\n",
       " u'over',\n",
       " u'under',\n",
       " u'again',\n",
       " u'further',\n",
       " u'then',\n",
       " u'once',\n",
       " u'here',\n",
       " u'there',\n",
       " u'when',\n",
       " u'where',\n",
       " u'why',\n",
       " u'how',\n",
       " u'all',\n",
       " u'any',\n",
       " u'both',\n",
       " u'each',\n",
       " u'few',\n",
       " u'more',\n",
       " u'most',\n",
       " u'other',\n",
       " u'some',\n",
       " u'such',\n",
       " u'no',\n",
       " u'nor',\n",
       " u'not',\n",
       " u'only',\n",
       " u'own',\n",
       " u'same',\n",
       " u'so',\n",
       " u'than',\n",
       " u'too',\n",
       " u'very',\n",
       " u's',\n",
       " u't',\n",
       " u'can',\n",
       " u'will',\n",
       " u'just',\n",
       " u'don',\n",
       " u\"don't\",\n",
       " u'should',\n",
       " u\"should've\",\n",
       " u'now',\n",
       " u'd',\n",
       " u'll',\n",
       " u'm',\n",
       " u'o',\n",
       " u're',\n",
       " u've',\n",
       " u'y',\n",
       " u'ain',\n",
       " u'aren',\n",
       " u\"aren't\",\n",
       " u'couldn',\n",
       " u\"couldn't\",\n",
       " u'didn',\n",
       " u\"didn't\",\n",
       " u'doesn',\n",
       " u\"doesn't\",\n",
       " u'hadn',\n",
       " u\"hadn't\",\n",
       " u'hasn',\n",
       " u\"hasn't\",\n",
       " u'haven',\n",
       " u\"haven't\",\n",
       " u'isn',\n",
       " u\"isn't\",\n",
       " u'ma',\n",
       " u'mightn',\n",
       " u\"mightn't\",\n",
       " u'mustn',\n",
       " u\"mustn't\",\n",
       " u'needn',\n",
       " u\"needn't\",\n",
       " u'shan',\n",
       " u\"shan't\",\n",
       " u'shouldn',\n",
       " u\"shouldn't\",\n",
       " u'wasn',\n",
       " u\"wasn't\",\n",
       " u'weren',\n",
       " u\"weren't\",\n",
       " u'won',\n",
       " u\"won't\",\n",
       " u'wouldn',\n",
       " u\"wouldn't\"]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def text_processing(mess):\n",
    "    nopunc = [char for char in mess if char not in string.punctuation]\n",
    "    \n",
    "    nopunc = ''.join(nopunc)\n",
    "    \n",
    "    return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [Go, jurong, point, crazy, Available, bugis, n...\n",
       "1                       [Ok, lar, Joking, wif, u, oni]\n",
       "2    [Free, entry, 2, wkly, comp, win, FA, Cup, fin...\n",
       "3        [U, dun, say, early, hor, U, c, already, say]\n",
       "4    [Nah, dont, think, goes, usf, lives, around, t...\n",
       "Name: message, dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages['message'].head().apply(text_processing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:6: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "bow_transformer = CountVectorizer(analyzer=text_processing).fit(messages['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11425\n"
     ]
    }
   ],
   "source": [
    "print(len(bow_transformer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mess4 = messages['message'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mess4\n",
    "bow4 = bow_transformer.transform([mess4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 11425)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow4.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 4068)\t2\n",
      "  (0, 4629)\t1\n",
      "  (0, 5261)\t1\n",
      "  (0, 6204)\t1\n",
      "  (0, 6222)\t1\n",
      "  (0, 7186)\t1\n",
      "  (0, 9554)\t2\n"
     ]
    }
   ],
   "source": [
    "print(bow4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'say'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_transformer.get_feature_names()[9554]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:6: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "messages_bow =bow_transformer.transform(messages['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5572, 11425)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages_bow.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50548"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages_bow.nnz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparsity = 100.0*messages_bow.nnz/(messages_bow.shape[0]*messages_bow.shape[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07940295412668218"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf_transformer = TfidfTransformer().fit(messages_bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf4 = tfidf_transformer.transform(bow4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 9554)\t0.538562626293\n",
      "  (0, 7186)\t0.438936565338\n",
      "  (0, 6222)\t0.318721689295\n",
      "  (0, 6204)\t0.299537997237\n",
      "  (0, 5261)\t0.297299574059\n",
      "  (0, 4629)\t0.266198019061\n",
      "  (0, 4068)\t0.408325899334\n"
     ]
    }
   ],
   "source": [
    "print(tfidf4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.527076498901426"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking Inverse document freq. for a one word\n",
    "tfidf_transformer.idf_[bow_transformer.vocabulary_['university']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "messages_tfidf = tfidf_transformer.transform(messages_bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5572, 11425)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "spam_detect_model = MultinomialNB().fit(messages_tfidf, messages['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham'"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_detect_model.predict(tfidf4)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_pred = spam_detect_model.predict(messages_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ham', 'ham', 'spam', ..., 'ham', 'ham', 'ham'],\n",
       "      dtype='|S4')"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "msg_train,msg_test,label_train,label_test = train_test_split(messages['message'],\n",
    "                                                            messages['label'], \n",
    "                                                            test_size = 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "    ('bow', CountVectorizer(analyzer=text_processing)),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('classifier', MultinomialNB())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:6: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('bow', CountVectorizer(analyzer=<function text_processing at 0x0000000019AF1C88>,\n",
       "        binary=False, decode_error=u'strict', dtype=<type 'numpy.int64'>,\n",
       "        encoding=u'utf-8', input=u'content', lowercase=True, max_df=1.0,\n",
       "        max_features=None, min_df=1, ngram_range=(1, 1), prepro...         use_idf=True)), ('classifier', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(msg_train, label_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:6: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "predictions = pipeline.predict(msg_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.95      1.00      0.98      1436\n",
      "       spam       1.00      0.69      0.82       236\n",
      "\n",
      "avg / total       0.96      0.96      0.95      1672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(label_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1436    0]\n",
      " [  73  163]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(label_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Lets fit a Random forest classifier\n",
    "pipeline2 = Pipeline([\n",
    "    ('bow', CountVectorizer(analyzer=text_processing)),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('classifier', RandomForestClassifier())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:6: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('bow', CountVectorizer(analyzer=<function text_processing at 0x0000000019AF1C88>,\n",
       "        binary=False, decode_error=u'strict', dtype=<type 'numpy.int64'>,\n",
       "        encoding=u'utf-8', input=u'content', lowercase=True, max_df=1.0,\n",
       "        max_features=None, min_df=1, ngram_range=(1, 1), prepro...imators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline2.fit(msg_train, label_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:6: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.95      1.00      0.98      1436\n",
      "       spam       0.98      0.71      0.82       236\n",
      "\n",
      "avg / total       0.96      0.96      0.95      1672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions2 = pipeline2.predict(msg_test)\n",
    "print(classification_report(label_test, predictions2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}