gfg_4_gridsearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.metrics import classification_report, confusion_matrix, recall_score, roc_auc_score, accuracy_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import nltk\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.corpus import wordnet\n",
    "from nltk import wordpunct_tokenize\n",
    "from nltk import WordNetLemmatizer\n",
    "from nltk import sent_tokenize\n",
    "from nltk import pos_tag\n",
    "\n",
    "from src.util import identity, model_report, regtokenize\n",
    "from src.transformers import GildPreprocessor, WordCounterSubjectivity, WordCounter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = 50 # The weight for the positive (minority) class\n",
    "\n",
    "# With word count feature\n",
    "model3 = Pipeline([\n",
    "    ('preprocess_union', FeatureUnion([\n",
    "    \t\t('term_freq_feature', Pipeline([\n",
    "    \t\t\t('normalizer', GildPreprocessor()),\n",
    "    \t\t\t('vectorizer', TfidfVectorizer(tokenizer=identity, preprocessor=None, lowercase=False))\n",
    "    \t\t\t])),\n",
    "    \t\t('word_counter', WordCounter())\n",
    "    \t])),\n",
    "    ('trees', RandomForestClassifier(random_state=0, n_jobs=-1, class_weight={0: 1, 1: w}))\n",
    "])\n",
    "\n",
    "# With word count & subjectivity score feature\n",
    "model4 = Pipeline([\n",
    "    ('preprocess_union', FeatureUnion([\n",
    "    \t\t('term_freq_feature', Pipeline([\n",
    "    \t\t\t('normalizer', GildPreprocessor()),\n",
    "    \t\t\t('vectorizer', TfidfVectorizer(tokenizer=identity, preprocessor=None, lowercase=False))\n",
    "    \t\t\t])),\n",
    "    \t\t('word_counter', WordCounterSubjectivity())\n",
    "    \t])),\n",
    "    ('trees', RandomForestClassifier(random_state=0, n_jobs=-1, class_weight={0: 1, 1: w}))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/askreddit_top_50.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Top-level comment set : ~9% minority class\n",
    "X = df['body']\n",
    "y = df['target']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Baseline run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda3/lib/python3.7/site-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('preprocess_union',\n",
       "                 FeatureUnion(n_jobs=None,\n",
       "                              transformer_list=[('term_freq_feature',\n",
       "                                                 Pipeline(memory=None,\n",
       "                                                          steps=[('normalizer',\n",
       "                                                                  GildPreprocessor(lower=True,\n",
       "                                                                                   strip=True)),\n",
       "                                                                 ('vectorizer',\n",
       "                                                                  TfidfVectorizer(analyzer='word',\n",
       "                                                                                  binary=False,\n",
       "                                                                                  decode_error='strict',\n",
       "                                                                                  dtype=<class 'numpy.float64'>,\n",
       "                                                                                  encoding='utf-8',\n",
       "                                                                                  input='content',\n",
       "                                                                                  lowercase=Fa...\n",
       "                 RandomForestClassifier(bootstrap=True,\n",
       "                                        class_weight={0: 1, 1: 50},\n",
       "                                        criterion='gini', max_depth=None,\n",
       "                                        max_features='auto',\n",
       "                                        max_leaf_nodes=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=1, min_samples_split=2,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=10, n_jobs=-1,\n",
       "                                        oob_score=False, random_state=0,\n",
       "                                        verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model4.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0   0.910387  0.983498  0.945531       909\n",
      "           1   0.166667  0.032967  0.055046        91\n",
      "\n",
      "    accuracy                       0.897000      1000\n",
      "   macro avg   0.538527  0.508233  0.500289      1000\n",
      "weighted avg   0.842708  0.897000  0.864497      1000\n",
      "\n",
      "Classes predicted:   [0 1]\n",
      "Accuracy:   0.897\n",
      "ROC AUC Score:   0.6487324556631487\n",
      "True Pos:  3 False Pos:  15\n",
      "FP to TP Ratio::   5.0\n",
      "True Neg:  894 False Neg:  88\n"
     ]
    }
   ],
   "source": [
    "model_report(model4, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = GridSearchCV(model4, param_grid={\n",
    "    'preprocess_union__term_freq_feature__vectorizer__ngram_range': [(1, 1), (1, 2), (1, 3)],\n",
    "    'trees__n_estimators': [10, 50, 100],\n",
    "}, scoring='precision')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "//anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv='warn', error_score='raise-deprecating',\n",
       "             estimator=Pipeline(memory=None,\n",
       "                                steps=[('preprocess_union',\n",
       "                                        FeatureUnion(n_jobs=None,\n",
       "                                                     transformer_list=[('term_freq_feature',\n",
       "                                                                        Pipeline(memory=None,\n",
       "                                                                                 steps=[('normalizer',\n",
       "                                                                                         GildPreprocessor(lower=True,\n",
       "                                                                                                          strip=True)),\n",
       "                                                                                        ('vectorizer',\n",
       "                                                                                         TfidfVectorizer(analyzer='word',\n",
       "                                                                                                         binary=False,\n",
       "                                                                                                         decode_error='strict',\n",
       "                                                                                                         dtype=<cl...\n",
       "                                                               min_weight_fraction_leaf=0.0,\n",
       "                                                               n_estimators=10,\n",
       "                                                               n_jobs=-1,\n",
       "                                                               oob_score=False,\n",
       "                                                               random_state=0,\n",
       "                                                               verbose=0,\n",
       "                                                               warm_start=False))],\n",
       "                                verbose=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'preprocess_union__term_freq_feature__vectorizer__ngram_range': [(1,\n",
       "                                                                                           1),\n",
       "                                                                                          (1,\n",
       "                                                                                           2),\n",
       "                                                                                          (1,\n",
       "                                                                                           3)],\n",
       "                         'trees__n_estimators': [10, 50, 100]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='precision', verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.28894166666666665"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvres = pd.DataFrame(search.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_preprocess_union__term_freq_feature__vectorizer__ngram_range</th>\n",
       "      <th>param_trees__n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.321460</td>\n",
       "      <td>0.581647</td>\n",
       "      <td>4.108285</td>\n",
       "      <td>0.163185</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>10</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>0.117263</td>\n",
       "      <td>0.018631</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.431424</td>\n",
       "      <td>0.338721</td>\n",
       "      <td>4.109115</td>\n",
       "      <td>0.154141</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>50</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.225496</td>\n",
       "      <td>0.058624</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.021349</td>\n",
       "      <td>0.094532</td>\n",
       "      <td>3.918089</td>\n",
       "      <td>0.238590</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>100</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.250031</td>\n",
       "      <td>0.090021</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.091803</td>\n",
       "      <td>0.581536</td>\n",
       "      <td>3.908885</td>\n",
       "      <td>0.150554</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>10</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.288942</td>\n",
       "      <td>0.149916</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.406243</td>\n",
       "      <td>0.226818</td>\n",
       "      <td>3.842981</td>\n",
       "      <td>0.239762</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>50</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8.386229</td>\n",
       "      <td>0.279473</td>\n",
       "      <td>3.726926</td>\n",
       "      <td>0.219988</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>100</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.892056</td>\n",
       "      <td>0.170424</td>\n",
       "      <td>3.718215</td>\n",
       "      <td>0.206701</td>\n",
       "      <td>(1, 3)</td>\n",
       "      <td>10</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.166625</td>\n",
       "      <td>0.235688</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9.037331</td>\n",
       "      <td>0.152220</td>\n",
       "      <td>4.059880</td>\n",
       "      <td>0.424411</td>\n",
       "      <td>(1, 3)</td>\n",
       "      <td>50</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.782123</td>\n",
       "      <td>0.251414</td>\n",
       "      <td>4.180034</td>\n",
       "      <td>0.048772</td>\n",
       "      <td>(1, 3)</td>\n",
       "      <td>100</td>\n",
       "      <td>{'preprocess_union__term_freq_feature__vectori...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0       8.321460      0.581647         4.108285        0.163185   \n",
       "1       8.431424      0.338721         4.109115        0.154141   \n",
       "2       8.021349      0.094532         3.918089        0.238590   \n",
       "3       8.091803      0.581536         3.908885        0.150554   \n",
       "4       8.406243      0.226818         3.842981        0.239762   \n",
       "5       8.386229      0.279473         3.726926        0.219988   \n",
       "6       7.892056      0.170424         3.718215        0.206701   \n",
       "7       9.037331      0.152220         4.059880        0.424411   \n",
       "8       9.782123      0.251414         4.180034        0.048772   \n",
       "\n",
       "  param_preprocess_union__term_freq_feature__vectorizer__ngram_range  \\\n",
       "0                                             (1, 1)                   \n",
       "1                                             (1, 1)                   \n",
       "2                                             (1, 1)                   \n",
       "3                                             (1, 2)                   \n",
       "4                                             (1, 2)                   \n",
       "5                                             (1, 2)                   \n",
       "6                                             (1, 3)                   \n",
       "7                                             (1, 3)                   \n",
       "8                                             (1, 3)                   \n",
       "\n",
       "  param_trees__n_estimators  \\\n",
       "0                        10   \n",
       "1                        50   \n",
       "2                       100   \n",
       "3                        10   \n",
       "4                        50   \n",
       "5                       100   \n",
       "6                        10   \n",
       "7                        50   \n",
       "8                       100   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "0  {'preprocess_union__term_freq_feature__vectori...           0.130435   \n",
       "1  {'preprocess_union__term_freq_feature__vectori...           0.272727   \n",
       "2  {'preprocess_union__term_freq_feature__vectori...           0.375000   \n",
       "3  {'preprocess_union__term_freq_feature__vectori...           0.500000   \n",
       "4  {'preprocess_union__term_freq_feature__vectori...           0.000000   \n",
       "5  {'preprocess_union__term_freq_feature__vectori...           0.000000   \n",
       "6  {'preprocess_union__term_freq_feature__vectori...           0.000000   \n",
       "7  {'preprocess_union__term_freq_feature__vectori...           0.000000   \n",
       "8  {'preprocess_union__term_freq_feature__vectori...           0.000000   \n",
       "\n",
       "   split1_test_score  split2_test_score  mean_test_score  std_test_score  \\\n",
       "0           0.090909           0.130435         0.117263        0.018631   \n",
       "1           0.142857           0.260870         0.225496        0.058624   \n",
       "2           0.166667           0.208333         0.250031        0.090021   \n",
       "3           0.200000           0.166667         0.288942        0.149916   \n",
       "4           0.000000           0.000000         0.000000        0.000000   \n",
       "5           0.000000           0.000000         0.000000        0.000000   \n",
       "6           0.000000           0.500000         0.166625        0.235688   \n",
       "7           0.000000           0.000000         0.000000        0.000000   \n",
       "8           0.000000           0.000000         0.000000        0.000000   \n",
       "\n",
       "   rank_test_score  \n",
       "0                5  \n",
       "1                3  \n",
       "2                2  \n",
       "3                1  \n",
       "4                6  \n",
       "5                6  \n",
       "6                4  \n",
       "7                6  \n",
       "8                6  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvmodel = search.best_estimator_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('preprocess_union',\n",
       "                 FeatureUnion(n_jobs=None,\n",
       "                              transformer_list=[('term_freq_feature',\n",
       "                                                 Pipeline(memory=None,\n",
       "                                                          steps=[('normalizer',\n",
       "                                                                  GildPreprocessor(lower=True,\n",
       "                                                                                   strip=True)),\n",
       "                                                                 ('vectorizer',\n",
       "                                                                  TfidfVectorizer(analyzer='word',\n",
       "                                                                                  binary=False,\n",
       "                                                                                  decode_error='strict',\n",
       "                                                                                  dtype=<class 'numpy.float64'>,\n",
       "                                                                                  encoding='utf-8',\n",
       "                                                                                  input='content',\n",
       "                                                                                  lowercase=Fa...\n",
       "                 RandomForestClassifier(bootstrap=True,\n",
       "                                        class_weight={0: 1, 1: 50},\n",
       "                                        criterion='gini', max_depth=None,\n",
       "                                        max_features='auto',\n",
       "                                        max_leaf_nodes=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=1, min_samples_split=2,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=10, n_jobs=-1,\n",
       "                                        oob_score=False, random_state=0,\n",
       "                                        verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvmodel.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0   0.908818  0.997800  0.951232       909\n",
      "           1   0.000000  0.000000  0.000000        91\n",
      "\n",
      "    accuracy                       0.907000      1000\n",
      "   macro avg   0.454409  0.498900  0.475616      1000\n",
      "weighted avg   0.826115  0.907000  0.864670      1000\n",
      "\n",
      "Classes predicted:   [0 1]\n",
      "Accuracy:   0.907\n",
      "ROC AUC Score:   0.5918591859185919\n",
      "True Pos:  0 False Pos:  2\n",
      "FP to TP Ratio::   inf\n",
      "True Neg:  907 False Neg:  91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sweet/dev/gal/go-for-gold/src/util.py:34: RuntimeWarning: divide by zero encountered in long_scalars\n",
      "  print(\"FP to TP Ratio::  \", (fp/tp))\n"
     ]
    }
   ],
   "source": [
    "model_report(cvmodel, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}