MOUD_Text_Video_Production.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Video analysis on the MOUD dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains a model to evaluate facial behaviors from videos from the MOUD dataset obtaining and processing data obtained from OpenFace toolkit. LINK: https://github.com/TadasBaltrusaitis/OpenFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The path of the train and test transcriptions\n",
    "# The data is seperated in an 80-20 ratio and the test directory is untouched. \n",
    "train_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\VideoReviews\\transcriptions\\train\\*.csv\"\n",
    "test_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\VideoReviews\\transcriptions\\test\\*.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# funcion to combine multiple speech, annotation columns to one and drop rest of columns\n",
    "def clean_moud(df_name):\n",
    "    if 'Speech' not in df_name.columns:\n",
    "        df_name['Speech'] = ''    \n",
    "    if 'speech' in df_name.columns:\n",
    "        df_name['Speech'] = df_name[['Speech','speech']].fillna('').sum(axis=1)   \n",
    "    if 'transcription' in df_name.columns:\n",
    "        df_name['Speech'] = df_name[['Speech','transcription']].fillna('').sum(axis=1)\n",
    "\n",
    "    if 'sentimentAnnotation' not in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = 0    \n",
    "    if 'sentimentAnnotations' in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = df_name[['sentimentAnnotation','sentimentAnnotations']].fillna(0).sum(axis=1)\n",
    "    if 'sentimentannotations' in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = df_name[['sentimentAnnotation','sentimentannotations']].fillna(0).sum(axis=1)\n",
    "    \n",
    "    return df_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# funcion to append all utterances to dataframe\n",
    "def create_data_df(df_name,data_path):\n",
    "    '''\n",
    "    Returns a text dataframe with two columns 'Speech' and 'sentimentAnnotation'\n",
    "    Returns a sparse matrix of video features to be combined with the text tfidf later'\n",
    "    '''\n",
    "    # Creating video df\n",
    "    v_cols = []\n",
    "    skeleton_path = r'C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Python\\MOUD\\Text_Video\\video_skeleton.csv'\n",
    "    df_v = pd.DataFrame(pd.read_csv(skeleton_path, sep = ','))\n",
    "    df_v = df_v.drop([df_v.columns.values[0]],axis=1)\n",
    "\n",
    "    for f in glob.glob(data_path):\n",
    "        \n",
    "        # TEXT \n",
    "        # append speech utterances to text dataframe \n",
    "        df_name = df_name.append(pd.read_csv(f,sep=';'),ignore_index=True)\n",
    "        \n",
    "        # VIDEO\n",
    "        # Create sparse video matrix for each file consecutively while creating text dataframe\n",
    "        # It is done at this particular point to extract time related groups before the starttime and endtimes are lost\n",
    "        \n",
    "        # Creating a temporary text df to get times and clean\n",
    "        df_name_temp = pd.read_csv(f,sep=';')\n",
    "\n",
    "        df_name_temp = clean_moud(df_name_temp)\n",
    "        \n",
    "        # Remove neutral annotations\n",
    "        df_name_temp = df_name_temp.query('sentimentAnnotation != 0')\n",
    "         \n",
    "        # Creating a df of the corredponding OpenFace features file \n",
    "        v_name = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\OpenFaceFeatures\" + \"\\\\\" + f.rsplit(\"\\\\\",1)[1].split(\".\")[0] + \".mp4.csv\"\n",
    "        df_v_name = pd.read_csv(v_name, sep = \", \", engine = \"python\")\n",
    "    \n",
    "        # Splitting the video data by utterances\n",
    "        for starttime,endtime in zip(df_name_temp['#starttime'],df_name_temp['#endtime']):    \n",
    "            # Generate mean and standard deviation upto endtime of utterance, new df because columns need to be dropped\n",
    "            df_v_name_temp = df_v_name.query('timestamp >='+str(starttime)+'& timestamp <='+str(endtime)).agg(['mean','std'])\n",
    "            # Drop unwanted labels after querying because timestamp is required to filter in prev line\n",
    "            df_v_name_temp.drop(['frame','timestamp','confidence','success'], axis = 1)\n",
    "            # append single row of means and stds to the main dataframe\n",
    "            \n",
    "            df_v.loc[len(df_v)] = np.array(df_v_name_temp).ravel()\n",
    "\n",
    "    # TEXT \n",
    "    # combine multiple speech, annotation columns to one and drop rest of columns\n",
    "    df_name = clean_moud(df_name)\n",
    "    \n",
    "    # Remove neutral annotations\n",
    "    df_name = df_name.query('sentimentAnnotation != 0')\n",
    "    \n",
    "    df_name = df_name[['Speech','sentimentAnnotation']].reset_index(drop=True)  \n",
    "    \n",
    "    return df_name, df_v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frame_mean</th>\n",
       "      <th>timestamp_mean</th>\n",
       "      <th>confidence_mean</th>\n",
       "      <th>success_mean</th>\n",
       "      <th>gaze_0_x_mean</th>\n",
       "      <th>gaze_0_y_mean</th>\n",
       "      <th>gaze_0_z_mean</th>\n",
       "      <th>gaze_1_x_mean</th>\n",
       "      <th>gaze_1_y_mean</th>\n",
       "      <th>gaze_1_z_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>AU12_c_std</th>\n",
       "      <th>AU14_c_std</th>\n",
       "      <th>AU15_c_std</th>\n",
       "      <th>AU17_c_std</th>\n",
       "      <th>AU20_c_std</th>\n",
       "      <th>AU23_c_std</th>\n",
       "      <th>AU25_c_std</th>\n",
       "      <th>AU26_c_std</th>\n",
       "      <th>AU28_c_std</th>\n",
       "      <th>AU45_c_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>55.5</td>\n",
       "      <td>1.818485</td>\n",
       "      <td>0.981728</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.157933</td>\n",
       "      <td>0.228770</td>\n",
       "      <td>-0.960507</td>\n",
       "      <td>-0.067107</td>\n",
       "      <td>0.229786</td>\n",
       "      <td>-0.970847</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.134220</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.394816</td>\n",
       "      <td>0.260877</td>\n",
       "      <td>0.408521</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.209252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>199.0</td>\n",
       "      <td>6.606607</td>\n",
       "      <td>0.927676</td>\n",
       "      <td>0.932203</td>\n",
       "      <td>0.136627</td>\n",
       "      <td>0.213596</td>\n",
       "      <td>-0.964651</td>\n",
       "      <td>-0.072389</td>\n",
       "      <td>0.204741</td>\n",
       "      <td>-0.974137</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.195448</td>\n",
       "      <td>0.349260</td>\n",
       "      <td>0.461750</td>\n",
       "      <td>0.208327</td>\n",
       "      <td>0.381166</td>\n",
       "      <td>0.442871</td>\n",
       "      <td>0.365956</td>\n",
       "      <td>0.129447</td>\n",
       "      <td>0.496392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>357.0</td>\n",
       "      <td>11.878534</td>\n",
       "      <td>0.980742</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.147945</td>\n",
       "      <td>0.216788</td>\n",
       "      <td>-0.964866</td>\n",
       "      <td>-0.079685</td>\n",
       "      <td>0.213898</td>\n",
       "      <td>-0.973514</td>\n",
       "      <td>...</td>\n",
       "      <td>0.270928</td>\n",
       "      <td>0.498878</td>\n",
       "      <td>0.186892</td>\n",
       "      <td>0.485640</td>\n",
       "      <td>0.454361</td>\n",
       "      <td>0.186892</td>\n",
       "      <td>0.501795</td>\n",
       "      <td>0.500755</td>\n",
       "      <td>0.186892</td>\n",
       "      <td>0.427034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>521.5</td>\n",
       "      <td>17.367371</td>\n",
       "      <td>0.975868</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.166863</td>\n",
       "      <td>0.217611</td>\n",
       "      <td>-0.961535</td>\n",
       "      <td>-0.060091</td>\n",
       "      <td>0.236668</td>\n",
       "      <td>-0.969625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.365606</td>\n",
       "      <td>0.327050</td>\n",
       "      <td>0.483651</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.404961</td>\n",
       "      <td>0.314373</td>\n",
       "      <td>0.338926</td>\n",
       "      <td>0.499066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>657.5</td>\n",
       "      <td>21.905250</td>\n",
       "      <td>0.981475</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.158007</td>\n",
       "      <td>0.217275</td>\n",
       "      <td>-0.963203</td>\n",
       "      <td>-0.066873</td>\n",
       "      <td>0.227610</td>\n",
       "      <td>-0.971425</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.329243</td>\n",
       "      <td>0.398733</td>\n",
       "      <td>0.110432</td>\n",
       "      <td>0.188897</td>\n",
       "      <td>0.481047</td>\n",
       "      <td>0.495691</td>\n",
       "      <td>0.155207</td>\n",
       "      <td>0.416463</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 862 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   frame_mean  timestamp_mean  confidence_mean  success_mean  gaze_0_x_mean  \\\n",
       "0        55.5        1.818485         0.981728      1.000000       0.157933   \n",
       "1       199.0        6.606607         0.927676      0.932203       0.136627   \n",
       "2       357.0       11.878534         0.980742      1.000000       0.147945   \n",
       "3       521.5       17.367371         0.975868      1.000000       0.166863   \n",
       "4       657.5       21.905250         0.981475      1.000000       0.158007   \n",
       "\n",
       "   gaze_0_y_mean  gaze_0_z_mean  gaze_1_x_mean  gaze_1_y_mean  gaze_1_z_mean  \\\n",
       "0       0.228770      -0.960507      -0.067107       0.229786      -0.970847   \n",
       "1       0.213596      -0.964651      -0.072389       0.204741      -0.974137   \n",
       "2       0.216788      -0.964866      -0.079685       0.213898      -0.973514   \n",
       "3       0.217611      -0.961535      -0.060091       0.236668      -0.969625   \n",
       "4       0.217275      -0.963203      -0.066873       0.227610      -0.971425   \n",
       "\n",
       "      ...      AU12_c_std  AU14_c_std  AU15_c_std  AU17_c_std  AU20_c_std  \\\n",
       "0     ...        0.000000    0.000000    0.000000    0.134220    0.000000   \n",
       "1     ...        0.000000    0.195448    0.349260    0.461750    0.208327   \n",
       "2     ...        0.270928    0.498878    0.186892    0.485640    0.454361   \n",
       "3     ...        0.000000    0.000000    0.365606    0.327050    0.483651   \n",
       "4     ...        0.000000    0.000000    0.329243    0.398733    0.110432   \n",
       "\n",
       "   AU23_c_std  AU25_c_std  AU26_c_std  AU28_c_std  AU45_c_std  \n",
       "0    0.394816    0.260877    0.408521    0.000000    0.209252  \n",
       "1    0.381166    0.442871    0.365956    0.129447    0.496392  \n",
       "2    0.186892    0.501795    0.500755    0.186892    0.427034  \n",
       "3    0.000000    0.404961    0.314373    0.338926    0.499066  \n",
       "4    0.188897    0.481047    0.495691    0.155207    0.416463  \n",
       "\n",
       "[5 rows x 862 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df_t = pd.DataFrame()\n",
    "\n",
    "# Clean dataframe and create sparse video matrix\n",
    "df, v_train = create_data_df(df,train_path)\n",
    "df_t, v_test = create_data_df(df_t,test_path)\n",
    "\n",
    "#\n",
    "v_train_sparse = scipy.sparse.csr_matrix(v_train.values)\n",
    "v_test_sparse = scipy.sparse.csr_matrix(v_test.values)\n",
    "\n",
    "v_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data cleaning and text preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section 'Data cleaning and text preprocessing' is to preprocess the text for text+video analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from https://www.kaggle.com/c/word2vec-nlp-tutorial/\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "\n",
    "# execute the following commented step to install the data packages if you don't already have it  \n",
    "# nltk.download()\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "# resuable function to convert raw speech to preprocessed\n",
    "def utterance_to_words(raw_utterance):\n",
    "    # 1. Removing any HTML elements\n",
    "    utterance_text = BeautifulSoup(raw_utterance, \"lxml\").get_text()\n",
    "    # 2. Remove non-letters\n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", utterance_text) \n",
    "    # 3. Convert to lower case, split into individual words\n",
    "    words = letters_only.lower().split()\n",
    "    # 4. convert the stop words to a set\n",
    "    stops = set(stopwords.words(\"spanish\"))\n",
    "    # 5. Remove stop words\n",
    "    meaningful_words = [w for w in words if not w in stops]\n",
    "    # 6. Join the words back into one string separated by space, and return the result.\n",
    "    return( \" \".join( meaningful_words ))\n",
    "\n",
    "# applying the function to the speech column\n",
    "df['Speech'] = df['Speech'].apply(lambda x: utterance_to_words(x))\n",
    "df_t['Speech'] = df_t['Speech'].apply(lambda x: utterance_to_words(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speech</th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>habia visto resenas decian picaba usabas</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>verdad si use vez t arde asi usas arde ojo</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dije puede ser posible deseaba arde voy poder ...</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tambien tira poquito pelo hagan cuenta quebra ...</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>igual lavadas dejado tirar</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Speech  sentimentAnnotation\n",
       "0           habia visto resenas decian picaba usabas                 -1.0\n",
       "1         verdad si use vez t arde asi usas arde ojo                 -1.0\n",
       "2  dije puede ser posible deseaba arde voy poder ...                 -1.0\n",
       "3  tambien tira poquito pelo hagan cuenta quebra ...                 -1.0\n",
       "4                         igual lavadas dejado tirar                  1.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "\n",
    "# # splitting dataset into train and test in stratified fashion and a ratio of 80% - 20%\n",
    "# X, y = df[['Speech']],df[['sentimentAnnotation']]\n",
    "# X_trn, X_tst, y_trn, y_tst = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n",
    "X_trn, y_trn = df[['Speech']],df[['sentimentAnnotation']]\n",
    "X_tst, y_tst = df_t[['Speech']],df_t[['sentimentAnnotation']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Utterance level video-ONLY analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following 'video-only analysis' code is present here due to dependency of timestamps from text dataset.\n",
    "This section performs analysis on only the video features extracted. \n",
    "\n",
    "The next section 'Machine Learning' contains both the video and text stacked using the 'early fusion' method. (See section 6.1 https://arxiv.org/pdf/1705.09406.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "# SVM model creation and fitting train vector to annotations\n",
    "model_tf_v = svm.SVC(kernel='linear', C=1, gamma=1).fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "\n",
    "# generate predictions\n",
    "predicted_tf_v = model_tf_v.predict(v_test_sparse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.51      0.49      0.50        41\n",
      "          1       0.62      0.65      0.64        54\n",
      "\n",
      "avg / total       0.58      0.58      0.58        95\n",
      "\n",
      "Accuracy: 0.58 (+/- 0.00)\n",
      "Mean sentiment: 'Positive'. Predicted mean sentiment: 'Positive'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>Right/Wrong</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentimentAnnotation  Prediction  Right/Wrong\n",
       "0                    1         1.0         True\n",
       "1                    1        -1.0        False\n",
       "2                   -1         1.0        False\n",
       "3                    1         1.0         True\n",
       "4                   -1         1.0        False"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf_v))\n",
    "\n",
    "#create df to show results\n",
    "disp = y_tst.reset_index(drop=True).join(pd.DataFrame(predicted_tf_v,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "scores = model_tf_v.score(v_test_sparse,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}. Predicted mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative','Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.49 (+/- 0.26)\n"
     ]
    }
   ],
   "source": [
    "# cross validation of training set\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, v_train_sparse, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.57      0.56      0.57        41\n",
      "          1       0.67      0.69      0.68        54\n",
      "\n",
      "avg / total       0.63      0.63      0.63        95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "model_lr = LogisticRegression().fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_lr = model_lr.predict(v_test_sparse)\n",
    "# Classification report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_lr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.42      0.32      0.36        41\n",
      "          1       0.56      0.67      0.61        54\n",
      "\n",
      "avg / total       0.50      0.52      0.50        95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "model_dt = DecisionTreeClassifier().fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_dt = model_dt.predict(v_test_sparse)\n",
    "# Classification report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_dt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.46      0.54      0.49        41\n",
      "          1       0.60      0.52      0.55        54\n",
      "\n",
      "avg / total       0.54      0.53      0.53        95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "model_rf = RandomForestClassifier().fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_rf = model_rf.predict(v_test_sparse)\n",
    "# Classification report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.51      0.49      0.50        41\n",
      "          1       0.62      0.65      0.64        54\n",
      "\n",
      "avg / total       0.58      0.58      0.58        95\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.57      0.56      0.57        41\n",
      "          1       0.67      0.69      0.68        54\n",
      "\n",
      "avg / total       0.63      0.63      0.63        95\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.42      0.32      0.36        41\n",
      "          1       0.56      0.67      0.61        54\n",
      "\n",
      "avg / total       0.50      0.52      0.50        95\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.46      0.54      0.49        41\n",
      "          1       0.60      0.52      0.55        54\n",
      "\n",
      "avg / total       0.54      0.53      0.53        95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf_v))\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_lr))\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_dt))\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_rf))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RFE is used to select most useful features in the data. After studying the columns carefully we see that most of the features show us position of the face in the video (for eg. X, Y and Z co-ordinates which hinder the learning proess by providing the model misguided information (i.e. the model thinks that the position of the face matters in the sentiment and tries to generalize using that.) A new file with selected features like AU or Action Units are considered to provide a more robust and better model in 'Feature Selection' notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of features: 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{1: 'AU45_r_std',\n",
       " 2: 'AU01_c_std',\n",
       " 3: 'AU17_r_std',\n",
       " 4: 'AU23_c_mean',\n",
       " 5: 'AU09_r_std',\n",
       " 6: 'AU14_r_std',\n",
       " 7: 'AU04_r_mean',\n",
       " 8: 'Z_50_std',\n",
       " 9: 'pose_Tz_std',\n",
       " 10: 'AU28_c_mean',\n",
       " 11: 'p_18_mean',\n",
       " 12: 'p_9_std',\n",
       " 13: 'Y_4_std',\n",
       " 14: 'y_5_std',\n",
       " 15: 'y_56_std',\n",
       " 16: 'Y_7_std',\n",
       " 17: 'AU10_c_std',\n",
       " 18: 'y_64_std',\n",
       " 19: 'p_4_std',\n",
       " 20: 'y_9_std',\n",
       " 21: 'Y_31_std',\n",
       " 22: 'y_54_std',\n",
       " 23: 'p_11_mean',\n",
       " 24: 'x_35_mean',\n",
       " 25: 'x_24_mean',\n",
       " 26: 'X_45_mean',\n",
       " 27: 'X_58_mean',\n",
       " 28: 'AU25_c_mean',\n",
       " 29: 'Z_50_mean',\n",
       " 30: 'Z_8_mean',\n",
       " 31: 'y_65_std',\n",
       " 32: 'Y_60_mean',\n",
       " 33: 'Y_3_mean',\n",
       " 34: 'X_59_mean',\n",
       " 35: 'Y_21_std',\n",
       " 36: 'p_13_std',\n",
       " 37: 'x_4_std',\n",
       " 38: 'X_2_std',\n",
       " 39: 'x_31_std',\n",
       " 40: 'X_35_std',\n",
       " 41: 'x_0_std',\n",
       " 42: 'p_22_mean',\n",
       " 43: 'pose_Ty_std',\n",
       " 44: 'Y_8_std',\n",
       " 45: 'Z_57_std',\n",
       " 46: 'Z_0_std',\n",
       " 47: 'Z_31_std',\n",
       " 48: 'AU05_c_mean',\n",
       " 49: 'Z_41_std',\n",
       " 50: 'Z_20_std',\n",
       " 51: 'AU28_c_std',\n",
       " 52: 'y_4_std',\n",
       " 53: 'p_6_mean',\n",
       " 54: 'x_44_mean',\n",
       " 55: 'x_8_mean',\n",
       " 56: 'Y_10_mean',\n",
       " 57: 'AU04_c_std',\n",
       " 58: 'AU06_c_std',\n",
       " 59: 'y_16_mean',\n",
       " 60: 'x_25_mean',\n",
       " 61: 'y_36_mean',\n",
       " 62: 'x_34_mean',\n",
       " 63: 'Y_14_mean',\n",
       " 64: 'X_12_mean',\n",
       " 65: 'AU09_r_mean',\n",
       " 66: 'AU14_c_std',\n",
       " 67: 'p_3_mean',\n",
       " 68: 'x_46_mean',\n",
       " 69: 'p_6_std',\n",
       " 70: 'x_42_std',\n",
       " 71: 'x_25_std',\n",
       " 72: 'Z_22_std',\n",
       " 73: 'Z_24_std',\n",
       " 74: 'x_47_std',\n",
       " 75: 'AU45_r_mean',\n",
       " 76: 'Z_26_std',\n",
       " 77: 'Z_16_std',\n",
       " 78: 'AU14_r_mean',\n",
       " 79: 'X_42_std',\n",
       " 80: 'AU12_c_mean',\n",
       " 81: 'p_20_mean',\n",
       " 82: 'AU10_r_std',\n",
       " 83: 'pose_Tx_mean',\n",
       " 84: 'pose_Tz_mean',\n",
       " 85: 'AU10_c_mean',\n",
       " 86: 'Y_2_mean',\n",
       " 87: 'p_24_mean',\n",
       " 88: 'p_21_mean',\n",
       " 89: 'p_19_mean',\n",
       " 90: 'y_53_std',\n",
       " 91: 'p_14_std',\n",
       " 92: 'AU26_r_mean',\n",
       " 93: 'AU14_c_mean',\n",
       " 94: 'p_3_std',\n",
       " 95: 'p_0_std',\n",
       " 96: 'AU01_r_mean',\n",
       " 97: 'X_67_mean',\n",
       " 98: 'p_15_mean',\n",
       " 99: 'x_52_mean',\n",
       " 100: 'y_0_mean',\n",
       " 101: 'y_12_mean',\n",
       " 102: 'y_5_mean',\n",
       " 103: 'p_1_mean',\n",
       " 104: 'Z_49_std',\n",
       " 105: 'X_46_std',\n",
       " 106: 'X_25_std',\n",
       " 107: 'x_59_mean',\n",
       " 108: 'y_11_mean',\n",
       " 109: 'y_67_mean',\n",
       " 110: 'y_15_mean',\n",
       " 111: 'y_4_mean',\n",
       " 112: 'y_10_mean',\n",
       " 113: 'x_53_mean',\n",
       " 114: 'p_4_mean',\n",
       " 115: 'x_26_mean',\n",
       " 116: 'y_19_mean',\n",
       " 117: 'x_51_mean',\n",
       " 118: 'y_10_std',\n",
       " 119: 'AU01_c_mean',\n",
       " 120: 'X_57_mean',\n",
       " 121: 'pose_Tx_std',\n",
       " 122: 'x_7_mean',\n",
       " 123: 'y_25_mean',\n",
       " 124: 'X_60_mean',\n",
       " 125: 'X_21_mean',\n",
       " 126: 'X_4_mean',\n",
       " 127: 'X_0_mean',\n",
       " 128: 'X_47_std',\n",
       " 129: 'x_3_std',\n",
       " 130: 'Y_3_std',\n",
       " 131: 'Y_0_std',\n",
       " 132: 'Y_2_std',\n",
       " 133: 'AU25_r_mean',\n",
       " 134: 'x_50_std',\n",
       " 135: 'x_30_std',\n",
       " 136: 'y_41_mean',\n",
       " 137: 'Z_7_mean',\n",
       " 138: 'Z_55_mean',\n",
       " 139: 'Z_49_mean',\n",
       " 140: 'x_41_std',\n",
       " 141: 'Z_36_std',\n",
       " 142: 'Z_30_std',\n",
       " 143: 'X_14_mean',\n",
       " 144: 'Y_32_std',\n",
       " 145: 'p_16_mean',\n",
       " 146: 'p_11_std',\n",
       " 147: 'p_10_std',\n",
       " 148: 'AU04_c_mean',\n",
       " 149: 'X_53_std',\n",
       " 150: 'X_14_std',\n",
       " 151: 'X_16_std',\n",
       " 152: 'AU02_c_mean',\n",
       " 153: 'Z_61_std',\n",
       " 154: 'Y_65_std',\n",
       " 155: 'Y_58_std',\n",
       " 156: 'y_55_std',\n",
       " 157: 'Z_6_std',\n",
       " 158: 'Z_3_std',\n",
       " 159: 'y_3_std',\n",
       " 160: 'y_51_std',\n",
       " 161: 'y_37_std',\n",
       " 162: 'AU02_r_std',\n",
       " 163: 'X_66_mean',\n",
       " 164: 'y_59_mean',\n",
       " 165: 'Z_37_std',\n",
       " 166: 'p_21_std',\n",
       " 167: 'Z_29_std',\n",
       " 168: 'Y_20_std',\n",
       " 169: 'x_40_std',\n",
       " 170: 'x_24_std',\n",
       " 171: 'Y_59_std',\n",
       " 172: 'Y_26_std',\n",
       " 173: 'p_22_std',\n",
       " 174: 'y_52_std',\n",
       " 175: 'p_14_mean',\n",
       " 176: 'x_64_std',\n",
       " 177: 'Y_6_std',\n",
       " 178: 'Y_49_std',\n",
       " 179: 'p_15_std',\n",
       " 180: 'X_48_mean',\n",
       " 181: 'Y_67_mean',\n",
       " 182: 'p_12_mean',\n",
       " 183: 'Z_9_mean',\n",
       " 184: 'Z_12_mean',\n",
       " 185: 'success_std',\n",
       " 186: 'X_35_mean',\n",
       " 187: 'X_56_mean',\n",
       " 188: 'x_54_mean',\n",
       " 189: 'x_35_std',\n",
       " 190: 'Y_59_mean',\n",
       " 191: 'Y_41_mean',\n",
       " 192: 'p_23_mean',\n",
       " 193: 'p_10_mean',\n",
       " 194: 'y_22_mean',\n",
       " 195: 'y_20_mean',\n",
       " 196: 'y_6_std',\n",
       " 197: 'x_1_std',\n",
       " 198: 'X_55_std',\n",
       " 199: 'y_11_std',\n",
       " 200: 'Y_12_std',\n",
       " 201: 'Y_10_std',\n",
       " 202: 'X_43_std',\n",
       " 203: 'X_1_std',\n",
       " 204: 'X_17_std',\n",
       " 205: 'x_58_mean',\n",
       " 206: 'Z_51_mean',\n",
       " 207: 'X_20_mean',\n",
       " 208: 'x_47_mean',\n",
       " 209: 'Z_32_std',\n",
       " 210: 'x_50_mean',\n",
       " 211: 'x_21_mean',\n",
       " 212: 'p_13_mean',\n",
       " 213: 'X_51_mean',\n",
       " 214: 'Z_23_mean',\n",
       " 215: 'Y_43_std',\n",
       " 216: 'p_18_std',\n",
       " 217: 'Y_50_std',\n",
       " 218: 'Y_54_std',\n",
       " 219: 'p_16_std',\n",
       " 220: 'y_30_std',\n",
       " 221: 'y_66_std',\n",
       " 222: 'x_38_std',\n",
       " 223: 'AU07_r_mean',\n",
       " 224: 'y_2_std',\n",
       " 225: 'Y_17_std',\n",
       " 226: 'y_18_std',\n",
       " 227: 'Z_48_std',\n",
       " 228: 'X_13_mean',\n",
       " 229: 'y_59_std',\n",
       " 230: 'Y_9_mean',\n",
       " 231: 'X_65_mean',\n",
       " 232: 'Z_52_mean',\n",
       " 233: 'Z_19_mean',\n",
       " 234: 'Z_13_mean',\n",
       " 235: 'Z_25_mean',\n",
       " 236: 'p_5_mean',\n",
       " 237: 'y_57_std',\n",
       " 238: 'Y_34_std',\n",
       " 239: 'x_61_std',\n",
       " 240: 'y_50_std',\n",
       " 241: 'Z_2_std',\n",
       " 242: 'Z_1_std',\n",
       " 243: 'Y_39_mean',\n",
       " 244: 'Y_66_mean',\n",
       " 245: 'X_30_std',\n",
       " 246: 'x_5_std',\n",
       " 247: 'X_50_std',\n",
       " 248: 'Z_61_mean',\n",
       " 249: 'AU45_c_std',\n",
       " 250: 'x_45_mean',\n",
       " 251: 'x_56_mean',\n",
       " 252: 'Z_6_mean',\n",
       " 253: 'Z_26_mean',\n",
       " 254: 'x_23_mean',\n",
       " 255: 'Z_17_mean',\n",
       " 256: 'p_0_mean',\n",
       " 257: 'x_54_std',\n",
       " 258: 'X_50_mean',\n",
       " 259: 'x_16_mean',\n",
       " 260: 'y_27_mean',\n",
       " 261: 'y_30_mean',\n",
       " 262: 'y_6_mean',\n",
       " 263: 'y_17_mean',\n",
       " 264: 'AU01_r_std',\n",
       " 265: 'Y_60_std',\n",
       " 266: 'y_49_std',\n",
       " 267: 'AU25_c_std',\n",
       " 268: 'X_45_std',\n",
       " 269: 'x_57_mean',\n",
       " 270: 'x_64_mean',\n",
       " 271: 'Z_40_std',\n",
       " 272: 'Z_21_std',\n",
       " 273: 'Z_45_std',\n",
       " 274: 'Z_62_std',\n",
       " 275: 'Z_25_std',\n",
       " 276: 'X_24_std',\n",
       " 277: 'X_22_std',\n",
       " 278: 'Y_33_std',\n",
       " 279: 'Y_27_std',\n",
       " 280: 'Y_42_std',\n",
       " 281: 'x_37_std',\n",
       " 282: 'x_17_std',\n",
       " 283: 'x_43_std',\n",
       " 284: 'x_29_std',\n",
       " 285: 'x_16_std',\n",
       " 286: 'x_39_std',\n",
       " 287: 'y_29_std',\n",
       " 288: 'X_54_std',\n",
       " 289: 'y_43_std',\n",
       " 290: 'y_46_std',\n",
       " 291: 'AU20_c_std',\n",
       " 292: 'p_23_std',\n",
       " 293: 'AU17_c_std',\n",
       " 294: 'Z_20_mean',\n",
       " 295: 'y_37_mean',\n",
       " 296: 'Z_22_mean',\n",
       " 297: 'y_13_mean',\n",
       " 298: 'y_26_mean',\n",
       " 299: 'Y_4_mean',\n",
       " 300: 'Y_26_mean',\n",
       " 301: 'y_66_mean',\n",
       " 302: 'x_67_mean',\n",
       " 303: 'x_34_std',\n",
       " 304: 'x_63_mean',\n",
       " 305: 'AU25_r_std',\n",
       " 306: 'x_2_std',\n",
       " 307: 'x_46_std',\n",
       " 308: 'x_9_mean',\n",
       " 309: 'Z_56_std',\n",
       " 310: 'Z_5_std',\n",
       " 311: 'Z_43_std',\n",
       " 312: 'Z_4_std',\n",
       " 313: 'p_25_std',\n",
       " 314: 'Z_51_std',\n",
       " 315: 'AU12_r_std',\n",
       " 316: 'Z_46_std',\n",
       " 317: 'AU09_c_mean',\n",
       " 318: 'Z_56_mean',\n",
       " 319: 'Z_62_mean',\n",
       " 320: 'y_38_std',\n",
       " 321: 'x_62_mean',\n",
       " 322: 'x_66_mean',\n",
       " 323: 'X_15_mean',\n",
       " 324: 'x_58_std',\n",
       " 325: 'X_3_std',\n",
       " 326: 'X_7_std',\n",
       " 327: 'X_13_std',\n",
       " 328: 'Y_61_std',\n",
       " 329: 'p_33_mean',\n",
       " 330: 'X_55_mean',\n",
       " 331: 'AU23_r_mean',\n",
       " 332: 'Z_58_std',\n",
       " 333: 'X_58_std',\n",
       " 334: 'X_34_std',\n",
       " 335: 'X_31_std',\n",
       " 336: 'X_48_std',\n",
       " 337: 'x_49_std',\n",
       " 338: 'p_25_mean',\n",
       " 339: 'y_48_std',\n",
       " 340: 'Z_28_std',\n",
       " 341: 'Y_65_mean',\n",
       " 342: 'y_1_std',\n",
       " 343: 'y_18_mean',\n",
       " 344: 'X_19_std',\n",
       " 345: 'Y_14_std',\n",
       " 346: 'Y_16_std',\n",
       " 347: 'p_7_std',\n",
       " 348: 'Z_45_mean',\n",
       " 349: 'y_32_std',\n",
       " 350: 'y_63_mean',\n",
       " 351: 'y_60_mean',\n",
       " 352: 'Z_24_mean',\n",
       " 353: 'y_24_mean',\n",
       " 354: 'X_5_mean',\n",
       " 355: 'X_1_mean',\n",
       " 356: 'x_43_mean',\n",
       " 357: 'x_44_std',\n",
       " 358: 'x_26_std',\n",
       " 359: 'X_19_mean',\n",
       " 360: 'x_10_mean',\n",
       " 361: 'x_19_mean',\n",
       " 362: 'x_4_mean',\n",
       " 363: 'Y_40_mean',\n",
       " 364: 'AU05_c_std',\n",
       " 365: 'AU12_c_std',\n",
       " 366: 'X_44_std',\n",
       " 367: 'x_19_std',\n",
       " 368: 'Y_22_mean',\n",
       " 369: 'y_40_mean',\n",
       " 370: 'Z_57_mean',\n",
       " 371: 'Y_8_mean',\n",
       " 372: 'Y_6_mean',\n",
       " 373: 'Y_48_mean',\n",
       " 374: 'Y_63_mean',\n",
       " 375: 'Z_63_mean',\n",
       " 376: 'Y_11_mean',\n",
       " 377: 'Y_15_mean',\n",
       " 378: 'Y_13_std',\n",
       " 379: 'y_12_std',\n",
       " 380: 'x_60_std',\n",
       " 381: 'AU04_r_std',\n",
       " 382: 'X_61_mean',\n",
       " 383: 'X_60_std',\n",
       " 384: 'X_49_std',\n",
       " 385: 'x_48_mean',\n",
       " 386: 'Z_33_std',\n",
       " 387: 'p_5_std',\n",
       " 388: 'Y_5_std',\n",
       " 389: 'AU17_c_mean',\n",
       " 390: 'X_44_mean',\n",
       " 391: 'Z_44_mean',\n",
       " 392: 'x_65_mean',\n",
       " 393: 'X_39_mean',\n",
       " 394: 'X_65_std',\n",
       " 395: 'y_3_mean',\n",
       " 396: 'AU20_c_mean',\n",
       " 397: 'y_39_mean',\n",
       " 398: 'y_9_mean',\n",
       " 399: 'Z_15_std',\n",
       " 400: 'y_22_std',\n",
       " 401: 'y_28_std',\n",
       " 402: 'y_58_mean',\n",
       " 403: 'p_27_mean',\n",
       " 404: 'x_27_mean',\n",
       " 405: 'AU26_c_std',\n",
       " 406: 'x_30_mean',\n",
       " 407: 'y_47_std',\n",
       " 408: 'y_44_std',\n",
       " 409: 'y_15_std',\n",
       " 410: 'p_20_std',\n",
       " 411: 'Y_55_std',\n",
       " 412: 'Y_64_std',\n",
       " 413: 'Z_42_std',\n",
       " 414: 'X_11_std',\n",
       " 415: 'Y_45_std',\n",
       " 416: 'AU23_c_std',\n",
       " 417: 'Z_0_mean',\n",
       " 418: 'y_7_std',\n",
       " 419: 'p_8_std',\n",
       " 420: 'x_20_mean',\n",
       " 421: 'p_30_mean',\n",
       " 422: 'x_36_std',\n",
       " 423: 'Y_49_mean',\n",
       " 424: 'Y_62_mean',\n",
       " 425: 'Y_25_std',\n",
       " 426: 'AU02_r_mean',\n",
       " 427: 'y_44_mean',\n",
       " 428: 'y_38_mean',\n",
       " 429: 'y_19_std',\n",
       " 430: 'x_61_mean',\n",
       " 431: 'Z_5_mean',\n",
       " 432: 'y_48_mean',\n",
       " 433: 'Z_38_std',\n",
       " 434: 'AU12_r_mean',\n",
       " 435: 'x_55_std',\n",
       " 436: 'X_61_std',\n",
       " 437: 'X_62_mean',\n",
       " 438: 'y_35_std',\n",
       " 439: 'X_31_mean',\n",
       " 440: 'X_3_mean',\n",
       " 441: 'p_19_std',\n",
       " 442: 'X_4_std',\n",
       " 443: 'X_6_std',\n",
       " 444: 'Z_7_std',\n",
       " 445: 'X_26_std',\n",
       " 446: 'AU06_r_std',\n",
       " 447: 'p_26_mean',\n",
       " 448: 'AU45_c_mean',\n",
       " 449: 'Y_44_std',\n",
       " 450: 'AU15_c_std',\n",
       " 451: 'Z_53_mean',\n",
       " 452: 'X_38_std',\n",
       " 453: 'X_29_std',\n",
       " 454: 'y_33_std',\n",
       " 455: 'Y_38_std',\n",
       " 456: 'Y_36_std',\n",
       " 457: 'X_53_mean',\n",
       " 458: 'y_52_mean',\n",
       " 459: 'y_14_std',\n",
       " 460: 'y_16_std',\n",
       " 461: 'Y_21_mean',\n",
       " 462: 'Y_57_std',\n",
       " 463: 'X_46_mean',\n",
       " 464: 'x_22_mean',\n",
       " 465: 'y_67_std',\n",
       " 466: 'Z_43_mean',\n",
       " 467: 'Z_30_mean',\n",
       " 468: 'y_23_mean',\n",
       " 469: 'confidence_std',\n",
       " 470: 'y_21_mean',\n",
       " 471: 'X_32_std',\n",
       " 472: 'y_28_mean',\n",
       " 473: 'x_33_mean',\n",
       " 474: 'x_28_std',\n",
       " 475: 'X_41_mean',\n",
       " 476: 'Z_46_mean',\n",
       " 477: 'Z_16_mean',\n",
       " 478: 'y_26_std',\n",
       " 479: 'p_17_mean',\n",
       " 480: 'Y_18_std',\n",
       " 481: 'Y_56_std',\n",
       " 482: 'X_43_mean',\n",
       " 483: 'X_26_mean',\n",
       " 484: 'y_23_std',\n",
       " 485: 'y_60_std',\n",
       " 486: 'Y_53_std',\n",
       " 487: 'Y_22_std',\n",
       " 488: 'y_43_mean',\n",
       " 489: 'y_53_mean',\n",
       " 490: 'x_60_mean',\n",
       " 491: 'x_45_std',\n",
       " 492: 'X_27_mean',\n",
       " 493: 'X_11_mean',\n",
       " 494: 'Y_1_mean',\n",
       " 495: 'p_9_mean',\n",
       " 496: 'Y_48_std',\n",
       " 497: 'x_48_std',\n",
       " 498: 'y_62_std',\n",
       " 499: 'y_17_std',\n",
       " 500: 'Z_19_std',\n",
       " 501: 'Z_9_std',\n",
       " 502: 'x_6_mean',\n",
       " 503: 'x_3_mean',\n",
       " 504: 'x_1_mean',\n",
       " 505: 'Z_67_mean',\n",
       " 506: 'AU15_c_mean',\n",
       " 507: 'Z_58_mean',\n",
       " 508: 'Z_39_mean',\n",
       " 509: 'X_5_std',\n",
       " 510: 'Y_13_mean',\n",
       " 511: 'Z_64_std',\n",
       " 512: 'Z_60_std',\n",
       " 513: 'x_62_std',\n",
       " 514: 'y_62_mean',\n",
       " 515: 'Z_11_mean',\n",
       " 516: 'Z_17_std',\n",
       " 517: 'X_52_mean',\n",
       " 518: 'AU15_r_mean',\n",
       " 519: 'x_5_mean',\n",
       " 520: 'Y_23_mean',\n",
       " 521: 'X_36_mean',\n",
       " 522: 'X_18_mean',\n",
       " 523: 'y_63_std',\n",
       " 524: 'Y_61_mean',\n",
       " 525: 'x_57_std',\n",
       " 526: 'x_65_std',\n",
       " 527: 'y_8_std',\n",
       " 528: 'x_37_mean',\n",
       " 529: 'x_14_mean',\n",
       " 530: 'x_11_mean',\n",
       " 531: 'Z_32_mean',\n",
       " 532: 'Z_35_mean',\n",
       " 533: 'y_31_mean',\n",
       " 534: 'Y_66_std',\n",
       " 535: 'X_59_std',\n",
       " 536: 'Z_29_mean',\n",
       " 537: 'Z_33_mean',\n",
       " 538: 'x_20_std',\n",
       " 539: 'X_57_std',\n",
       " 540: 'X_52_std',\n",
       " 541: 'AU02_c_std',\n",
       " 542: 'Y_15_std',\n",
       " 543: 'AU15_r_std',\n",
       " 544: 'X_47_mean',\n",
       " 545: 'X_64_mean',\n",
       " 546: 'Z_64_mean',\n",
       " 547: 'x_42_mean',\n",
       " 548: 'x_63_std',\n",
       " 549: 'y_65_mean',\n",
       " 550: 'pose_Ty_mean',\n",
       " 551: 'Z_60_mean',\n",
       " 552: 'X_54_mean',\n",
       " 553: 'p_24_std',\n",
       " 554: 'y_58_std',\n",
       " 555: 'x_22_std',\n",
       " 556: 'x_0_mean',\n",
       " 557: 'x_17_mean',\n",
       " 558: 'p_26_std',\n",
       " 559: 'x_66_std',\n",
       " 560: 'X_32_mean',\n",
       " 561: 'y_27_std',\n",
       " 562: 'y_45_mean',\n",
       " 563: 'x_6_std',\n",
       " 564: 'x_7_std',\n",
       " 565: 'X_28_mean',\n",
       " 566: 'y_36_std',\n",
       " 567: 'Y_37_std',\n",
       " 568: 'y_13_std',\n",
       " 569: 'p_12_std',\n",
       " 570: 'X_40_mean',\n",
       " 571: 'p_17_std',\n",
       " 572: 'X_34_mean',\n",
       " 573: 'X_64_std',\n",
       " 574: 'Z_14_mean',\n",
       " 575: 'y_51_mean',\n",
       " 576: 'y_1_mean',\n",
       " 577: 'Y_17_mean',\n",
       " 578: 'p_8_mean',\n",
       " 579: 'Z_8_std',\n",
       " 580: 'y_24_std',\n",
       " 581: 'Y_35_std',\n",
       " 582: 'x_51_std',\n",
       " 583: 'AU23_r_std',\n",
       " 584: 'p_29_mean',\n",
       " 585: 'p_27_std',\n",
       " 586: 'Z_21_mean',\n",
       " 587: 'Z_28_mean',\n",
       " 588: 'X_10_mean',\n",
       " 589: 'x_33_std',\n",
       " 590: 'x_56_std',\n",
       " 591: 'Z_47_std',\n",
       " 592: 'Y_20_mean',\n",
       " 593: 'Y_27_mean',\n",
       " 594: 'x_36_mean',\n",
       " 595: 'x_39_mean',\n",
       " 596: 'X_40_std',\n",
       " 597: 'X_63_std',\n",
       " 598: 'y_25_std',\n",
       " 599: 'y_41_std',\n",
       " 600: 'Y_31_mean',\n",
       " 601: 'Y_58_mean',\n",
       " 602: 'AU05_r_std',\n",
       " 603: 'x_13_std',\n",
       " 604: 'Y_19_std',\n",
       " 605: 'y_47_mean',\n",
       " 606: 'y_7_mean',\n",
       " 607: 'Y_0_mean',\n",
       " 608: 'pose_Rx_std',\n",
       " 609: 'Z_48_mean',\n",
       " 610: 'Z_10_mean',\n",
       " 611: 'p_rx_std',\n",
       " 612: 'X_0_std',\n",
       " 613: 'Y_46_std',\n",
       " 614: 'y_46_mean',\n",
       " 615: 'x_27_std',\n",
       " 616: 'Y_25_mean',\n",
       " 617: 'Z_13_std',\n",
       " 618: 'Z_10_std',\n",
       " 619: 'Z_67_std',\n",
       " 620: 'Z_55_std',\n",
       " 621: 'AU17_r_mean',\n",
       " 622: 'x_32_std',\n",
       " 623: 'y_33_mean',\n",
       " 624: 'y_49_mean',\n",
       " 625: 'y_31_std',\n",
       " 626: 'Z_4_mean',\n",
       " 627: 'y_61_mean',\n",
       " 628: 'Z_23_std',\n",
       " 629: 'p_ty_std',\n",
       " 630: 'Y_36_mean',\n",
       " 631: 'Y_43_mean',\n",
       " 632: 'p_7_mean',\n",
       " 633: 'AU06_c_mean',\n",
       " 634: 'y_40_std',\n",
       " 635: 'y_35_mean',\n",
       " 636: 'x_29_mean',\n",
       " 637: 'x_8_std',\n",
       " 638: 'X_8_std',\n",
       " 639: 'X_21_std',\n",
       " 640: 'gaze_0_y_std',\n",
       " 641: 'x_55_mean',\n",
       " 642: 'success_mean',\n",
       " 643: 'Z_41_mean',\n",
       " 644: 'x_59_std',\n",
       " 645: 'X_42_mean',\n",
       " 646: 'X_25_mean',\n",
       " 647: 'Y_33_mean',\n",
       " 648: 'AU07_c_std',\n",
       " 649: 'p_28_mean',\n",
       " 650: 'X_38_mean',\n",
       " 651: 'Y_11_std',\n",
       " 652: 'y_42_mean',\n",
       " 653: 'Y_50_mean',\n",
       " 654: 'p_33_std',\n",
       " 655: 'y_39_std',\n",
       " 656: 'Y_32_mean',\n",
       " 657: 'Y_41_std',\n",
       " 658: 'p_30_std',\n",
       " 659: 'Y_44_mean',\n",
       " 660: 'x_18_std',\n",
       " 661: 'X_20_std',\n",
       " 662: 'x_12_std',\n",
       " 663: 'X_49_mean',\n",
       " 664: 'Z_11_std',\n",
       " 665: 'p_1_std',\n",
       " 666: 'AU20_r_mean',\n",
       " 667: 'x_13_mean',\n",
       " 668: 'timestamp_std',\n",
       " 669: 'Z_27_mean',\n",
       " 670: 'p_2_std',\n",
       " 671: 'Y_16_mean',\n",
       " 672: 'Z_44_std',\n",
       " 673: 'X_30_mean',\n",
       " 674: 'y_55_mean',\n",
       " 675: 'X_18_std',\n",
       " 676: 'X_29_mean',\n",
       " 677: 'X_12_std',\n",
       " 678: 'X_28_std',\n",
       " 679: 'Z_36_mean',\n",
       " 680: 'Z_14_std',\n",
       " 681: 'X_22_mean',\n",
       " 682: 'timestamp_mean',\n",
       " 683: 'Z_42_mean',\n",
       " 684: 'AU07_c_mean',\n",
       " 685: 'X_37_mean',\n",
       " 686: 'Y_56_mean',\n",
       " 687: 'confidence_mean',\n",
       " 688: 'Z_47_mean',\n",
       " 689: 'y_34_std',\n",
       " 690: 'x_28_mean',\n",
       " 691: 'y_57_mean',\n",
       " 692: 'X_9_mean',\n",
       " 693: 'X_16_mean',\n",
       " 694: 'Y_51_std',\n",
       " 695: 'x_40_mean',\n",
       " 696: 'AU06_r_mean',\n",
       " 697: 'x_15_std',\n",
       " 698: 'Z_54_mean',\n",
       " 699: 'AU07_r_std',\n",
       " 700: 'gaze_0_x_std',\n",
       " 701: 'AU05_r_mean',\n",
       " 702: 'x_18_mean',\n",
       " 703: 'X_63_mean',\n",
       " 704: 'Z_31_mean',\n",
       " 705: 'y_61_std',\n",
       " 706: 'X_67_std',\n",
       " 707: 'X_2_mean',\n",
       " 708: 'Y_47_mean',\n",
       " 709: 'Z_1_mean',\n",
       " 710: 'x_15_mean',\n",
       " 711: 'Y_54_mean',\n",
       " 712: 'gaze_1_y_std',\n",
       " 713: 'AU09_c_std',\n",
       " 714: 'Y_28_std',\n",
       " 715: 'p_rz_std',\n",
       " 716: 'p_28_std',\n",
       " 717: 'X_56_std',\n",
       " 718: 'X_66_std',\n",
       " 719: 'gaze_1_y_mean',\n",
       " 720: 'Y_42_mean',\n",
       " 721: 'Z_53_std',\n",
       " 722: 'p_31_mean',\n",
       " 723: 'y_8_mean',\n",
       " 724: 'Z_35_std',\n",
       " 725: 'X_33_std',\n",
       " 726: 'AU20_r_std',\n",
       " 727: 'pose_Rz_std',\n",
       " 728: 'X_17_mean',\n",
       " 729: 'x_12_mean',\n",
       " 730: 'x_21_std',\n",
       " 731: 'x_41_mean',\n",
       " 732: 'Z_3_mean',\n",
       " 733: 'y_50_mean',\n",
       " 734: 'p_2_mean',\n",
       " 735: 'Z_54_std',\n",
       " 736: 'x_67_std',\n",
       " 737: 'p_32_mean',\n",
       " 738: 'Y_24_std',\n",
       " 739: 'Y_35_mean',\n",
       " 740: 'Y_39_std',\n",
       " 741: 'y_29_mean',\n",
       " 742: 'x_10_std',\n",
       " 743: 'x_11_std',\n",
       " 744: 'X_10_std',\n",
       " 745: 'Y_28_mean',\n",
       " 746: 'Y_19_mean',\n",
       " 747: 'X_24_mean',\n",
       " 748: 'X_23_std',\n",
       " 749: 'Z_12_std',\n",
       " 750: 'Y_52_std',\n",
       " 751: 'pose_Rz_mean',\n",
       " 752: 'Y_57_mean',\n",
       " 753: 'y_56_mean',\n",
       " 754: 'X_27_std',\n",
       " 755: 'gaze_1_x_std',\n",
       " 756: 'Z_38_mean',\n",
       " 757: 'Z_65_std',\n",
       " 758: 'pose_Ry_mean',\n",
       " 759: 'Y_37_mean',\n",
       " 760: 'X_51_std',\n",
       " 761: 'p_32_std',\n",
       " 762: 'p_tx_std',\n",
       " 763: 'y_2_mean',\n",
       " 764: 'pose_Ry_std',\n",
       " 765: 'y_14_mean',\n",
       " 766: 'Z_65_mean',\n",
       " 767: 'X_15_std',\n",
       " 768: 'Z_39_std',\n",
       " 769: 'Y_9_std',\n",
       " 770: 'Y_45_mean',\n",
       " 771: 'Y_47_std',\n",
       " 772: 'AU10_r_mean',\n",
       " 773: 'X_37_std',\n",
       " 774: 'X_9_std',\n",
       " 775: 'y_34_mean',\n",
       " 776: 'X_39_std',\n",
       " 777: 'AU26_r_std',\n",
       " 778: 'X_36_std',\n",
       " 779: 'y_64_mean',\n",
       " 780: 'X_23_mean',\n",
       " 781: 'X_6_mean',\n",
       " 782: 'Y_30_mean',\n",
       " 783: 'p_ry_std',\n",
       " 784: 'x_2_mean',\n",
       " 785: 'Z_37_mean',\n",
       " 786: 'p_29_std',\n",
       " 787: 'x_14_std',\n",
       " 788: 'Z_59_mean',\n",
       " 789: 'Y_7_mean',\n",
       " 790: 'p_31_std',\n",
       " 791: 'Y_12_mean',\n",
       " 792: 'x_32_mean',\n",
       " 793: 'p_rz_mean',\n",
       " 794: 'gaze_0_y_mean',\n",
       " 795: 'x_52_std',\n",
       " 796: 'p_ry_mean',\n",
       " 797: 'Y_23_std',\n",
       " 798: 'gaze_1_x_mean',\n",
       " 799: 'gaze_0_x_mean',\n",
       " 800: 'x_9_std',\n",
       " 801: 'x_53_std',\n",
       " 802: 'Y_34_mean',\n",
       " 803: 'Y_1_std',\n",
       " 804: 'Z_34_mean',\n",
       " 805: 'y_0_std',\n",
       " 806: 'Y_62_std',\n",
       " 807: 'y_42_std',\n",
       " 808: 'x_49_mean',\n",
       " 809: 'X_62_std',\n",
       " 810: 'Z_66_mean',\n",
       " 811: 'X_8_mean',\n",
       " 812: 'x_31_mean',\n",
       " 813: 'Z_34_std',\n",
       " 814: 'Y_67_std',\n",
       " 815: 'Z_40_mean',\n",
       " 816: 'p_rx_mean',\n",
       " 817: 'y_45_std',\n",
       " 818: 'Y_52_mean',\n",
       " 819: 'Y_29_std',\n",
       " 820: 'gaze_0_z_mean',\n",
       " 821: 'Y_30_std',\n",
       " 822: 'Y_63_std',\n",
       " 823: 'p_scale_std',\n",
       " 824: 'X_41_std',\n",
       " 825: 'p_scale_mean',\n",
       " 826: 'pose_Rx_mean',\n",
       " 827: 'Y_40_std',\n",
       " 828: 'X_7_mean',\n",
       " 829: 'Z_66_std',\n",
       " 830: 'Z_15_mean',\n",
       " 831: 'Y_5_mean',\n",
       " 832: 'frame_mean',\n",
       " 833: 'Y_29_mean',\n",
       " 834: 'gaze_1_z_std',\n",
       " 835: 'Y_53_mean',\n",
       " 836: 'Y_55_mean',\n",
       " 837: 'frame_std',\n",
       " 838: 'Z_27_std',\n",
       " 839: 'AU26_c_mean',\n",
       " 840: 'X_33_mean',\n",
       " 841: 'Y_38_mean',\n",
       " 842: 'Z_52_std',\n",
       " 843: 'y_21_std',\n",
       " 844: 'gaze_1_z_mean',\n",
       " 845: 'gaze_0_z_std',\n",
       " 846: 'Y_64_mean',\n",
       " 847: 'Z_59_std',\n",
       " 848: 'y_32_mean',\n",
       " 849: 'Y_46_mean',\n",
       " 850: 'x_38_mean',\n",
       " 851: 'x_23_std',\n",
       " 852: 'y_54_mean',\n",
       " 853: 'p_tx_mean',\n",
       " 854: 'p_ty_mean',\n",
       " 855: 'Y_18_mean',\n",
       " 856: 'Y_24_mean',\n",
       " 857: 'y_20_std',\n",
       " 858: 'Y_51_mean',\n",
       " 859: 'Z_18_mean',\n",
       " 860: 'Z_2_mean',\n",
       " 861: 'Z_63_std',\n",
       " 862: 'Z_18_std'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RFE\n",
    "from sklearn.feature_selection import RFE\n",
    "rfe = RFE(model_lr,1)\n",
    "fit = rfe.fit(v_train,y_trn['sentimentAnnotation'].values)\n",
    "# print(\"Num of features:\",fit.n_features_)\n",
    "dict(zip(fit.ranking_,v_train.columns))\n",
    "# print(\"Selected Features\")\n",
    "# print(fit.support_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Machine learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# countVectorizer initialization\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(analyzer = \"word\",   \\\n",
    "                             tokenizer = None,    \\\n",
    "                             preprocessor = None, \\\n",
    "                             stop_words = None,   \\\n",
    "                             lowercase = True,    \\\n",
    "                             max_features = 5000) \n",
    "\n",
    "# create bag of words vector for the training set using countVectorizer\n",
    "train_data_features = vectorizer.fit_transform(X_trn['Speech'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# transformation of test data\n",
    "test_data_features = vectorizer.transform(X_tst['Speech'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf-idf transformer initialization\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "# create tfidf transformed vector for the training set using tf-idf transformer\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(train_data_features)\n",
    "X_test_tfidf = tfidf_transformer.transform(test_data_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Stacking Video to Text\n",
    "train_data_features_v = scipy.sparse.hstack([X_train_tfidf, v_train_sparse])\n",
    "test_data_features_v = scipy.sparse.hstack([X_test_tfidf, v_test_sparse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.54      0.51      0.53        41\n",
      "          1       0.64      0.67      0.65        54\n",
      "\n",
      "avg / total       0.60      0.60      0.60        95\n",
      "\n",
      "Accuracy: 0.60 (+/- 0.00)\n",
      "Mean sentiment: 'Positive'. Predicted mean sentiment: 'Positive'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speech</th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>Right/Wrong</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>verdad recomiendo</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>eh</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>leido nunca ningun libro zombies</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>peliculas tal tampoco suelen hacer mucha graci...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>verdad gusta hablando</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Speech  sentimentAnnotation  \\\n",
       "0                                  verdad recomiendo                    1   \n",
       "1                                                 eh                    1   \n",
       "2                   leido nunca ningun libro zombies                   -1   \n",
       "3  peliculas tal tampoco suelen hacer mucha graci...                    1   \n",
       "4                              verdad gusta hablando                   -1   \n",
       "\n",
       "   Prediction  Right/Wrong  \n",
       "0         1.0         True  \n",
       "1         1.0         True  \n",
       "2         1.0        False  \n",
       "3         1.0         True  \n",
       "4         1.0        False  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SVM model creation and fitting train vector to annotations\n",
    "from sklearn import svm\n",
    "model_tf = svm.SVC(kernel='linear', C=1, gamma=1).fit(train_data_features_v,y_trn['sentimentAnnotation'].values)\n",
    "\n",
    "# generate predictions\n",
    "predicted_tf = model_tf.predict(test_data_features_v)\n",
    "\n",
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf))\n",
    "\n",
    "#create df to show results\n",
    "disp = X_tst.join(y_tst).reset_index(drop=True).join(pd.DataFrame(predicted_tf,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "scores = model_tf.score(test_data_features_v,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}. Predicted mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative','Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.49 (+/- 0.26)\n"
     ]
    }
   ],
   "source": [
    "# cross validation of training set\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, train_data_features_v, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.57      0.56      0.57        41\n",
      "          1       0.67      0.69      0.68        54\n",
      "\n",
      "avg / total       0.63      0.63      0.63        95\n",
      "\n",
      "Accuracy: 0.63 (+/- 0.00)\n",
      "Mean sentiment: 'Positive'. Predicted mean sentiment: 'Positive'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speech</th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>Right/Wrong</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>verdad recomiendo</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>eh</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>leido nunca ningun libro zombies</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>peliculas tal tampoco suelen hacer mucha graci...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>verdad gusta hablando</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Speech  sentimentAnnotation  \\\n",
       "0                                  verdad recomiendo                    1   \n",
       "1                                                 eh                    1   \n",
       "2                   leido nunca ningun libro zombies                   -1   \n",
       "3  peliculas tal tampoco suelen hacer mucha graci...                    1   \n",
       "4                              verdad gusta hablando                   -1   \n",
       "\n",
       "   Prediction  Right/Wrong  \n",
       "0         1.0         True  \n",
       "1        -1.0        False  \n",
       "2         1.0        False  \n",
       "3         1.0         True  \n",
       "4         1.0        False  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LR model creation and fitting train vector to annotations\n",
    "model_tf = LogisticRegression().fit(train_data_features_v,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_tf = model_tf.predict(test_data_features_v)\n",
    "\n",
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf))\n",
    "\n",
    "#create df to show results\n",
    "disp = X_tst.join(y_tst).reset_index(drop=True).join(pd.DataFrame(predicted_tf,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "scores = model_tf.score(test_data_features_v,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}. Predicted mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative','Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.64 (+/- 0.15)\n"
     ]
    }
   ],
   "source": [
    "# cross validation of training set\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, X_train_tfidf, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}