experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import accuracy_score\n",
    "from tpot import TPOTClassifier\n",
    "\n",
    "\n",
    "npf = pd.read_csv('data/npf_train.csv')\n",
    "\n",
    "npf.drop(\"partlybad\", axis=1, inplace=True)\n",
    "npf.set_index(\"date\", inplace=True)\n",
    "npf[\"class4\"] = npf[\"class4\"].astype(\"category\")\n",
    "npf.drop(\"id\", axis=1, inplace=True)\n",
    "class2 = np.array([\"event\"]*npf.shape[0],dtype=\"object\")\n",
    "class2[npf[\"class4\"]==\"nonevent\"] = \"nonevent\"\n",
    "npf.insert(0, 'class2', class2)\n",
    "npf[\"class2\"] = npf[\"class2\"].astype(\"category\")\n",
    "npf.insert(2, \"year\", pd.DatetimeIndex(npf.index).year)\n",
    "npf.insert(2, \"month\", pd.DatetimeIndex(npf.index).month)\n",
    "npf.insert(2, \"event\", pd.get_dummies(npf[\"class2\"])[\"event\"])\n",
    "\n",
    "monthmap = {1:1, 2:3, 3:10, 4:12, 5:11, 6:9, 7:4, 8:7, 9:8, 10:5, 11:2, 12:6} # months ordered descending by event rate\n",
    "npf[\"month\"] = npf.month.map(monthmap)\n",
    "\n",
    "classmap = {'nonevent':0, 'II':1, 'Ib':2, 'Ia':3}\n",
    "npf.insert(2, \"y4\", npf.class4.map(classmap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class2</th>\n",
       "      <th>class4</th>\n",
       "      <th>y4</th>\n",
       "      <th>event</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "      <th>CO2168.mean</th>\n",
       "      <th>CO2168.std</th>\n",
       "      <th>CO2336.mean</th>\n",
       "      <th>CO2336.std</th>\n",
       "      <th>...</th>\n",
       "      <th>T672.mean</th>\n",
       "      <th>T672.std</th>\n",
       "      <th>T84.mean</th>\n",
       "      <th>T84.std</th>\n",
       "      <th>UV_A.mean</th>\n",
       "      <th>UV_A.std</th>\n",
       "      <th>UV_B.mean</th>\n",
       "      <th>UV_B.std</th>\n",
       "      <th>CS.mean</th>\n",
       "      <th>CS.std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2000-01-01</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2000</td>\n",
       "      <td>384.462000</td>\n",
       "      <td>2.284996</td>\n",
       "      <td>384.164462</td>\n",
       "      <td>2.135062</td>\n",
       "      <td>...</td>\n",
       "      <td>-13.016471</td>\n",
       "      <td>0.525698</td>\n",
       "      <td>-12.422972</td>\n",
       "      <td>0.376324</td>\n",
       "      <td>1.635563</td>\n",
       "      <td>0.856948</td>\n",
       "      <td>0.026438</td>\n",
       "      <td>0.014617</td>\n",
       "      <td>0.003374</td>\n",
       "      <td>0.000733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-20</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2000</td>\n",
       "      <td>374.884615</td>\n",
       "      <td>0.415185</td>\n",
       "      <td>374.703333</td>\n",
       "      <td>0.385179</td>\n",
       "      <td>...</td>\n",
       "      <td>-8.997430</td>\n",
       "      <td>0.373927</td>\n",
       "      <td>-8.351043</td>\n",
       "      <td>0.575679</td>\n",
       "      <td>1.441109</td>\n",
       "      <td>0.741088</td>\n",
       "      <td>0.022649</td>\n",
       "      <td>0.012479</td>\n",
       "      <td>0.001501</td>\n",
       "      <td>0.000572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-23</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2000</td>\n",
       "      <td>373.496585</td>\n",
       "      <td>0.189497</td>\n",
       "      <td>373.382593</td>\n",
       "      <td>0.172958</td>\n",
       "      <td>...</td>\n",
       "      <td>-10.224472</td>\n",
       "      <td>0.965988</td>\n",
       "      <td>-9.651155</td>\n",
       "      <td>1.238891</td>\n",
       "      <td>2.677545</td>\n",
       "      <td>1.261612</td>\n",
       "      <td>0.044759</td>\n",
       "      <td>0.023748</td>\n",
       "      <td>0.000764</td>\n",
       "      <td>0.000048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-02-17</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2000</td>\n",
       "      <td>378.600367</td>\n",
       "      <td>1.934180</td>\n",
       "      <td>378.464862</td>\n",
       "      <td>1.946536</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.535183</td>\n",
       "      <td>0.122651</td>\n",
       "      <td>-0.829524</td>\n",
       "      <td>0.134191</td>\n",
       "      <td>2.261805</td>\n",
       "      <td>1.345651</td>\n",
       "      <td>0.030893</td>\n",
       "      <td>0.021903</td>\n",
       "      <td>0.002038</td>\n",
       "      <td>0.000751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-03-25</th>\n",
       "      <td>event</td>\n",
       "      <td>Ib</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>2000</td>\n",
       "      <td>373.128684</td>\n",
       "      <td>1.096617</td>\n",
       "      <td>372.980000</td>\n",
       "      <td>1.047750</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.095641</td>\n",
       "      <td>1.695622</td>\n",
       "      <td>-1.095864</td>\n",
       "      <td>2.090111</td>\n",
       "      <td>12.906779</td>\n",
       "      <td>7.022300</td>\n",
       "      <td>0.333523</td>\n",
       "      <td>0.239981</td>\n",
       "      <td>0.000662</td>\n",
       "      <td>0.000210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-08-14</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2011</td>\n",
       "      <td>377.541538</td>\n",
       "      <td>6.391551</td>\n",
       "      <td>377.802756</td>\n",
       "      <td>6.187754</td>\n",
       "      <td>...</td>\n",
       "      <td>16.470062</td>\n",
       "      <td>2.142789</td>\n",
       "      <td>17.077060</td>\n",
       "      <td>2.779125</td>\n",
       "      <td>16.129744</td>\n",
       "      <td>12.203500</td>\n",
       "      <td>0.786032</td>\n",
       "      <td>0.696306</td>\n",
       "      <td>0.002360</td>\n",
       "      <td>0.000253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-08-16</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2011</td>\n",
       "      <td>381.016623</td>\n",
       "      <td>4.411571</td>\n",
       "      <td>381.030844</td>\n",
       "      <td>4.062115</td>\n",
       "      <td>...</td>\n",
       "      <td>16.319361</td>\n",
       "      <td>1.089563</td>\n",
       "      <td>17.268471</td>\n",
       "      <td>1.308728</td>\n",
       "      <td>8.688739</td>\n",
       "      <td>8.320799</td>\n",
       "      <td>0.464422</td>\n",
       "      <td>0.496816</td>\n",
       "      <td>0.002423</td>\n",
       "      <td>0.000425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-08-18</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2011</td>\n",
       "      <td>386.687895</td>\n",
       "      <td>12.065627</td>\n",
       "      <td>386.284079</td>\n",
       "      <td>11.751004</td>\n",
       "      <td>...</td>\n",
       "      <td>13.315270</td>\n",
       "      <td>0.511146</td>\n",
       "      <td>13.781909</td>\n",
       "      <td>0.838839</td>\n",
       "      <td>7.375727</td>\n",
       "      <td>5.115708</td>\n",
       "      <td>0.366155</td>\n",
       "      <td>0.316015</td>\n",
       "      <td>0.001993</td>\n",
       "      <td>0.000391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-08-21</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2011</td>\n",
       "      <td>379.279128</td>\n",
       "      <td>12.045785</td>\n",
       "      <td>379.363087</td>\n",
       "      <td>11.533353</td>\n",
       "      <td>...</td>\n",
       "      <td>14.421092</td>\n",
       "      <td>1.696894</td>\n",
       "      <td>15.094141</td>\n",
       "      <td>1.745526</td>\n",
       "      <td>8.099394</td>\n",
       "      <td>5.851942</td>\n",
       "      <td>0.416961</td>\n",
       "      <td>0.363890</td>\n",
       "      <td>0.003484</td>\n",
       "      <td>0.000457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-08-22</th>\n",
       "      <td>nonevent</td>\n",
       "      <td>nonevent</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2011</td>\n",
       "      <td>384.443758</td>\n",
       "      <td>6.413297</td>\n",
       "      <td>384.364392</td>\n",
       "      <td>5.781036</td>\n",
       "      <td>...</td>\n",
       "      <td>15.049728</td>\n",
       "      <td>0.969001</td>\n",
       "      <td>16.076217</td>\n",
       "      <td>1.197644</td>\n",
       "      <td>11.665070</td>\n",
       "      <td>10.062797</td>\n",
       "      <td>0.622831</td>\n",
       "      <td>0.595032</td>\n",
       "      <td>0.004782</td>\n",
       "      <td>0.001082</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>458 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              class2    class4 y4  event  month  year  CO2168.mean  \\\n",
       "date                                                                 \n",
       "2000-01-01  nonevent  nonevent  0      0      1  2000   384.462000   \n",
       "2000-01-20  nonevent  nonevent  0      0      1  2000   374.884615   \n",
       "2000-01-23  nonevent  nonevent  0      0      1  2000   373.496585   \n",
       "2000-02-17  nonevent  nonevent  0      0      3  2000   378.600367   \n",
       "2000-03-25     event        Ib  2      1     10  2000   373.128684   \n",
       "...              ...       ... ..    ...    ...   ...          ...   \n",
       "2011-08-14  nonevent  nonevent  0      0      7  2011   377.541538   \n",
       "2011-08-16  nonevent  nonevent  0      0      7  2011   381.016623   \n",
       "2011-08-18  nonevent  nonevent  0      0      7  2011   386.687895   \n",
       "2011-08-21  nonevent  nonevent  0      0      7  2011   379.279128   \n",
       "2011-08-22  nonevent  nonevent  0      0      7  2011   384.443758   \n",
       "\n",
       "            CO2168.std  CO2336.mean  CO2336.std  ...  T672.mean  T672.std  \\\n",
       "date                                             ...                        \n",
       "2000-01-01    2.284996   384.164462    2.135062  ... -13.016471  0.525698   \n",
       "2000-01-20    0.415185   374.703333    0.385179  ...  -8.997430  0.373927   \n",
       "2000-01-23    0.189497   373.382593    0.172958  ... -10.224472  0.965988   \n",
       "2000-02-17    1.934180   378.464862    1.946536  ...  -1.535183  0.122651   \n",
       "2000-03-25    1.096617   372.980000    1.047750  ...  -2.095641  1.695622   \n",
       "...                ...          ...         ...  ...        ...       ...   \n",
       "2011-08-14    6.391551   377.802756    6.187754  ...  16.470062  2.142789   \n",
       "2011-08-16    4.411571   381.030844    4.062115  ...  16.319361  1.089563   \n",
       "2011-08-18   12.065627   386.284079   11.751004  ...  13.315270  0.511146   \n",
       "2011-08-21   12.045785   379.363087   11.533353  ...  14.421092  1.696894   \n",
       "2011-08-22    6.413297   384.364392    5.781036  ...  15.049728  0.969001   \n",
       "\n",
       "             T84.mean   T84.std  UV_A.mean   UV_A.std  UV_B.mean  UV_B.std  \\\n",
       "date                                                                         \n",
       "2000-01-01 -12.422972  0.376324   1.635563   0.856948   0.026438  0.014617   \n",
       "2000-01-20  -8.351043  0.575679   1.441109   0.741088   0.022649  0.012479   \n",
       "2000-01-23  -9.651155  1.238891   2.677545   1.261612   0.044759  0.023748   \n",
       "2000-02-17  -0.829524  0.134191   2.261805   1.345651   0.030893  0.021903   \n",
       "2000-03-25  -1.095864  2.090111  12.906779   7.022300   0.333523  0.239981   \n",
       "...               ...       ...        ...        ...        ...       ...   \n",
       "2011-08-14  17.077060  2.779125  16.129744  12.203500   0.786032  0.696306   \n",
       "2011-08-16  17.268471  1.308728   8.688739   8.320799   0.464422  0.496816   \n",
       "2011-08-18  13.781909  0.838839   7.375727   5.115708   0.366155  0.316015   \n",
       "2011-08-21  15.094141  1.745526   8.099394   5.851942   0.416961  0.363890   \n",
       "2011-08-22  16.076217  1.197644  11.665070  10.062797   0.622831  0.595032   \n",
       "\n",
       "             CS.mean    CS.std  \n",
       "date                            \n",
       "2000-01-01  0.003374  0.000733  \n",
       "2000-01-20  0.001501  0.000572  \n",
       "2000-01-23  0.000764  0.000048  \n",
       "2000-02-17  0.002038  0.000751  \n",
       "2000-03-25  0.000662  0.000210  \n",
       "...              ...       ...  \n",
       "2011-08-14  0.002360  0.000253  \n",
       "2011-08-16  0.002423  0.000425  \n",
       "2011-08-18  0.001993  0.000391  \n",
       "2011-08-21  0.003484  0.000457  \n",
       "2011-08-22  0.004782  0.001082  \n",
       "\n",
       "[458 rows x 106 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "npf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_bin = npf.event.values\n",
    "y_multi = npf.y4.values\n",
    "X_unscaled = npf.values[:,4:]\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X_unscaled)\n",
    "X = scaler.transform(X_unscaled)\n",
    "features = list(npf.columns)[4:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(366, 102)\n",
      "(92, 102)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y_multi, test_size=0.2)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Selection\n",
    "\n",
    "https://towardsdatascience.com/feature-selection-using-python-for-classification-problem-b5f00a1c7028 \n",
    "\n",
    "PCA, look at components that explain most of the variance in the model\n",
    "Univariate feature selection, compares each feature to the target and outputs the statistic. Then we pick the best ones. Can do this with chi-squared, f-test, Mutual Information (dependency)\n",
    "Can use SelectFromModel function which considers all features at once.\n",
    "Can use \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for univariate feature selection, using both chi_2 and f_statistic as our scores\n",
    "chi_2_featureSelections = []\n",
    "f_stat_featureSelections = []\n",
    "\n",
    "numFeatures = [10,20,50,70,90,102]\n",
    "\n",
    "for num in numFeatures:\n",
    "    chi_2_featureSelections.append(SelectKBest(chi2, k=num).fit_transform(X_train, y_train))\n",
    "    f_stat_featureSelections.append(SelectKBest(f_classif, k=num).fit_transform(X_train, y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If using RandomForest Classifier we can do feature selection with bootstrap sampling and assigned feature_importance scores. Then do recursive feature selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['O384.std', 'month', 'H2O336.std', 'NO504.mean', 'H2O168.std', 'H2O42.std', 'NO672.mean', 'O3168.std', 'O3504.std', 'SWS.std', 'H2O504.std', 'NO42.mean', 'O3504.mean', 'SO2168.std', 'O342.std'] \n",
      "\n",
      "['T168.std', 'NOx504.mean', 'H2O168.std', 'RPAR.std', 'month', 'T84.mean', 'O3168.std', 'O3672.std', 'NO42.mean', 'H2O42.std', 'O342.std', 'T672.std', 'T504.std', 'CO2168.std', 'NOx504.std'] \n",
      "\n",
      "['CO242.std', 'Glob.std', 'H2O84.std', 'NO42.std', 'NO672.mean', 'NOx168.std', 'NOx672.mean', 'NOx84.std', 'PTG.mean', 'RPAR.mean', 'RPAR.std', 'T42.std', 'T504.std', 'T672.std', 'PAR.std'] \n",
      "\n",
      "['month', 'O3504.std', 'T504.std', 'NO336.mean', 'T168.std', 'O342.std', 'PTG.mean', 'H2O336.std', 'H2O42.std', 'O3168.std', 'NO504.mean', 'RPAR.std', 'CO2336.mean', 'NOx504.std', 'H2O84.std'] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_estimators = [500,50,10,100]\n",
    "max_depth = [100,10,5,None]\n",
    "min_sample_split = [16,8,4,2]\n",
    "#i = 3 are default values\n",
    "for i in range(4):\n",
    "    rf = RandomForestClassifier(random_state=0,n_estimators=n_estimators[i],max_depth=max_depth[i],min_samples_split=min_sample_split[i])\n",
    "    rf.fit(X_train,y_train)\n",
    "    f_i = list(zip(features,rf.feature_importances_))\n",
    "    f_i.sort(key = lambda x : x[1])\n",
    "    f_i = [x[0] for x in f_i]\n",
    "    print(f_i[0:15],\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0.6413043478260869, 50, 100, 8), (0.6413043478260869, 50, None, 8), (0.6195652173913043, 10, 100, 4), (0.6195652173913043, 10, 100, 2), (0.6195652173913043, 10, None, 4), (0.6195652173913043, 10, None, 2), (0.6086956521739131, 10, 10, 8), (0.6086956521739131, 10, 10, 4), (0.6086956521739131, 100, 10, 8), (0.5978260869565217, 500, 100, 4), (0.5978260869565217, 500, 10, 8), (0.5978260869565217, 500, 10, 4), (0.5978260869565217, 500, 10, 2), (0.5978260869565217, 500, None, 4), (0.5978260869565217, 50, 100, 2), (0.5978260869565217, 50, 10, 16), (0.5978260869565217, 50, 10, 8), (0.5978260869565217, 50, 10, 4), (0.5978260869565217, 50, 10, 2), (0.5978260869565217, 50, None, 2), (0.5978260869565217, 100, 100, 8), (0.5978260869565217, 100, 100, 4), (0.5978260869565217, 100, 100, 2), (0.5978260869565217, 100, 10, 4), (0.5978260869565217, 100, None, 8), (0.5978260869565217, 100, None, 4), (0.5978260869565217, 100, None, 2), (0.5869565217391305, 500, 100, 16), (0.5869565217391305, 500, 100, 8), (0.5869565217391305, 500, 10, 16), (0.5869565217391305, 500, 5, 8), (0.5869565217391305, 500, 5, 4), (0.5869565217391305, 500, 5, 2), (0.5869565217391305, 500, None, 16), (0.5869565217391305, 500, None, 8), (0.5869565217391305, 50, 100, 16), (0.5869565217391305, 50, 100, 4), (0.5869565217391305, 50, 5, 4), (0.5869565217391305, 50, None, 16), (0.5869565217391305, 50, None, 4), (0.5869565217391305, 10, 100, 8), (0.5869565217391305, 10, None, 8), (0.5869565217391305, 100, 10, 16), (0.5869565217391305, 100, 10, 2), (0.5869565217391305, 100, 5, 16), (0.5869565217391305, 100, 5, 4), (0.5869565217391305, 100, 5, 2), (0.5760869565217391, 500, 5, 16), (0.5760869565217391, 50, 5, 16), (0.5760869565217391, 50, 5, 8), (0.5760869565217391, 10, 100, 16), (0.5760869565217391, 10, 10, 2), (0.5760869565217391, 10, None, 16), (0.5760869565217391, 100, 100, 16), (0.5760869565217391, 100, 5, 8), (0.5760869565217391, 100, None, 16), (0.5652173913043478, 500, 100, 2), (0.5652173913043478, 500, None, 2), (0.5652173913043478, 50, 5, 2), (0.5652173913043478, 10, 5, 8), (0.5543478260869565, 10, 10, 16), (0.5543478260869565, 10, 5, 16), (0.5543478260869565, 10, 5, 2), (0.532608695652174, 10, 5, 4)]\n"
     ]
    }
   ],
   "source": [
    "#note that this block took 2h23min to complete\n",
    "results1 = []\n",
    "for n_est in n_estimators:\n",
    "    for m_dep in max_depth:\n",
    "        for min_sam in min_sample_split:\n",
    "            rf = RandomForestClassifier(random_state=0,n_estimators=n_est,max_depth=m_dep,min_samples_split=min_sam)\n",
    "            rf.fit(X_train,y_train)\n",
    "            rfe = RFECV(rf,cv=5,scoring=\"accuracy\")\n",
    "            rfe.fit(X_train,y_train)\n",
    "            selected_features = np.array(features)[rfe.get_support()]\n",
    "            #print(selected_features)\n",
    "            y_pred_rfecv = rfe.predict(X_test)\n",
    "            #print(f\"n_estimators = {n_est}, max_depth = {m_dep}, min_samples_split = {min_sam}\\n\")\n",
    "            #print(\"\\nAccuracy:\",accuracy_score(y_test, y_pred_rfecv))\n",
    "            tup = (accuracy_score(y_test, y_pred_rfecv),n_est,m_dep,min_sam)\n",
    "            results1.append(tup)\n",
    "results1.sort(key = lambda x: x[0],reverse = True)\n",
    "print(results1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now using PCA for feature selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0.717391304347826, 10, 10, 10, 2), (0.6847826086956522, 10, 500, 10, 2), (0.6847826086956522, 10, 10, 10, 16), (0.6739130434782609, 10, 500, 100, 8), (0.6739130434782609, 10, 500, 100, 4), (0.6739130434782609, 10, 500, 10, 4), (0.6739130434782609, 10, 500, None, 8), (0.6739130434782609, 10, 500, None, 4), (0.6739130434782609, 10, 10, 100, 4), (0.6739130434782609, 10, 10, None, 4), (0.6739130434782609, 10, 100, 100, 16), (0.6739130434782609, 10, 100, 10, 4), (0.6739130434782609, 10, 100, 5, 4), (0.6739130434782609, 10, 100, None, 16), (0.6630434782608695, 10, 500, 100, 16), (0.6630434782608695, 10, 500, 100, 2), (0.6630434782608695, 10, 500, 10, 16), (0.6630434782608695, 10, 500, 10, 8), (0.6630434782608695, 10, 500, None, 16), (0.6630434782608695, 10, 500, None, 2), (0.6630434782608695, 10, 50, 10, 16), (0.6630434782608695, 10, 50, 10, 2), (0.6630434782608695, 10, 10, 10, 8), (0.6630434782608695, 10, 100, 10, 16), (0.6630434782608695, 10, 100, 5, 2), (0.6521739130434783, 10, 500, 5, 16), (0.6521739130434783, 10, 500, 5, 8), (0.6521739130434783, 10, 500, 5, 4), (0.6521739130434783, 10, 500, 5, 2), (0.6521739130434783, 10, 50, 100, 16), (0.6521739130434783, 10, 50, 100, 4), (0.6521739130434783, 10, 50, 10, 8), (0.6521739130434783, 10, 50, 10, 4), (0.6521739130434783, 10, 50, 5, 2), (0.6521739130434783, 10, 50, None, 16), (0.6521739130434783, 10, 50, None, 4), (0.6521739130434783, 10, 10, 100, 16), (0.6521739130434783, 10, 10, 100, 2), (0.6521739130434783, 10, 10, None, 16), (0.6521739130434783, 10, 10, None, 2), (0.6521739130434783, 10, 100, 100, 4), (0.6521739130434783, 10, 100, 10, 2), (0.6521739130434783, 10, 100, 5, 16), (0.6521739130434783, 10, 100, 5, 8), (0.6521739130434783, 10, 100, None, 4), (0.6413043478260869, 10, 50, 100, 8), (0.6413043478260869, 10, 50, 100, 2), (0.6413043478260869, 20, 50, 10, 4), (0.6413043478260869, 10, 50, 5, 16), (0.6413043478260869, 10, 50, 5, 4), (0.6413043478260869, 10, 50, None, 8), (0.6413043478260869, 10, 50, None, 2), (0.6413043478260869, 10, 100, 100, 8), (0.6413043478260869, 10, 100, None, 8), (0.6304347826086957, 50, 500, 100, 8), (0.6304347826086957, 50, 500, None, 8), (0.6304347826086957, 20, 50, 100, 4), (0.6304347826086957, 20, 50, None, 4), (0.6304347826086957, 10, 10, 100, 8), (0.6304347826086957, 10, 10, 10, 4), (0.6304347826086957, 10, 10, 5, 2), (0.6304347826086957, 10, 10, None, 8), (0.6304347826086957, 50, 100, 100, 8), (0.6304347826086957, 20, 100, 10, 4), (0.6304347826086957, 50, 100, None, 8), (0.6195652173913043, 20, 500, 100, 4), (0.6195652173913043, 20, 500, 10, 2), (0.6195652173913043, 20, 500, None, 4), (0.6195652173913043, 50, 50, 100, 8), (0.6195652173913043, 20, 50, 10, 2), (0.6195652173913043, 50, 50, None, 8), (0.6195652173913043, 10, 10, 5, 16), (0.6195652173913043, 50, 100, 10, 4), (0.6086956521739131, 20, 500, 100, 8), (0.6086956521739131, 20, 500, 100, 2), (0.6086956521739131, 50, 500, 100, 2), (0.6086956521739131, 20, 500, 10, 8), (0.6086956521739131, 50, 500, 10, 8), (0.6086956521739131, 20, 500, None, 8), (0.6086956521739131, 20, 500, None, 2), (0.6086956521739131, 50, 500, None, 2), (0.6086956521739131, 70, 50, 100, 4), (0.6086956521739131, 20, 50, 100, 2), (0.6086956521739131, 50, 50, 100, 2), (0.6086956521739131, 50, 50, 10, 4), (0.6086956521739131, 10, 50, 5, 8), (0.6086956521739131, 70, 50, None, 4), (0.6086956521739131, 20, 50, None, 2), (0.6086956521739131, 50, 50, None, 2), (0.6086956521739131, 20, 10, 100, 16), (0.6086956521739131, 70, 10, 100, 4), (0.6086956521739131, 20, 10, 10, 16), (0.6086956521739131, 10, 10, 5, 8), (0.6086956521739131, 20, 10, None, 16), (0.6086956521739131, 70, 10, None, 4), (0.6086956521739131, 20, 100, 100, 4), (0.6086956521739131, 50, 100, 100, 4), (0.6086956521739131, 10, 100, 100, 2), (0.6086956521739131, 10, 100, 10, 8), (0.6086956521739131, 20, 100, 10, 8), (0.6086956521739131, 20, 100, 10, 2), (0.6086956521739131, 20, 100, 5, 4), (0.6086956521739131, 20, 100, None, 4), (0.6086956521739131, 50, 100, None, 4), (0.6086956521739131, 10, 100, None, 2), (0.5978260869565217, 50, 500, 100, 16), (0.5978260869565217, 70, 500, 100, 8), (0.5978260869565217, 50, 500, 100, 4), (0.5978260869565217, 50, 500, 10, 16), (0.5978260869565217, 20, 500, 10, 4), (0.5978260869565217, 50, 500, None, 16), (0.5978260869565217, 70, 500, None, 8), (0.5978260869565217, 50, 500, None, 4), (0.5978260869565217, 20, 50, 100, 16), (0.5978260869565217, 70, 50, 100, 16), (0.5978260869565217, 50, 50, 100, 4), (0.5978260869565217, 50, 50, 10, 8), (0.5978260869565217, 70, 50, 10, 4), (0.5978260869565217, 20, 50, 5, 8), (0.5978260869565217, 20, 50, 5, 4), (0.5978260869565217, 20, 50, None, 16), (0.5978260869565217, 70, 50, None, 16), (0.5978260869565217, 50, 50, None, 4), (0.5978260869565217, 20, 10, 100, 2), (0.5978260869565217, 90, 10, 10, 8), (0.5978260869565217, 70, 10, 10, 4), (0.5978260869565217, 20, 10, None, 2), (0.5978260869565217, 70, 100, 100, 4), (0.5978260869565217, 20, 100, 100, 2), (0.5978260869565217, 70, 100, 10, 16), (0.5978260869565217, 70, 100, 10, 4), (0.5978260869565217, 20, 100, 5, 8), (0.5978260869565217, 20, 100, 5, 2), (0.5978260869565217, 70, 100, None, 4), (0.5978260869565217, 20, 100, None, 2), (0.5869565217391305, 70, 500, 100, 16), (0.5869565217391305, 50, 500, 10, 4), (0.5869565217391305, 50, 500, 10, 2), (0.5869565217391305, 70, 500, 10, 2), (0.5869565217391305, 20, 500, 5, 8), (0.5869565217391305, 70, 500, None, 16), (0.5869565217391305, 20, 50, 10, 8), (0.5869565217391305, 102, 50, 10, 4), (0.5869565217391305, 20, 50, 5, 2), (0.5869565217391305, 20, 10, 100, 4), (0.5869565217391305, 50, 10, 5, 8), (0.5869565217391305, 10, 10, 5, 4), (0.5869565217391305, 20, 10, 5, 4), (0.5869565217391305, 20, 10, None, 4), (0.5869565217391305, 20, 100, 100, 16), (0.5869565217391305, 50, 100, 100, 16), (0.5869565217391305, 20, 100, 100, 8), (0.5869565217391305, 50, 100, 100, 2), (0.5869565217391305, 70, 100, 100, 2), (0.5869565217391305, 50, 100, 10, 8), (0.5869565217391305, 50, 100, 10, 2), (0.5869565217391305, 20, 100, None, 16), (0.5869565217391305, 50, 100, None, 16), (0.5869565217391305, 20, 100, None, 8), (0.5869565217391305, 50, 100, None, 2), (0.5869565217391305, 70, 100, None, 2), (0.5760869565217391, 20, 500, 100, 16), (0.5760869565217391, 70, 500, 100, 2), (0.5760869565217391, 20, 500, 10, 16), (0.5760869565217391, 70, 500, 10, 16), (0.5760869565217391, 20, 500, 5, 16), (0.5760869565217391, 20, 500, 5, 4), (0.5760869565217391, 20, 500, 5, 2), (0.5760869565217391, 20, 500, None, 16), (0.5760869565217391, 70, 500, None, 2), (0.5760869565217391, 50, 50, 100, 16), (0.5760869565217391, 20, 50, 100, 8), (0.5760869565217391, 102, 50, 100, 4), (0.5760869565217391, 20, 50, 10, 16), (0.5760869565217391, 50, 50, 10, 16), (0.5760869565217391, 70, 50, 10, 16), (0.5760869565217391, 50, 50, 10, 2), (0.5760869565217391, 50, 50, None, 16), (0.5760869565217391, 20, 50, None, 8), (0.5760869565217391, 102, 50, None, 4), (0.5760869565217391, 50, 10, 100, 8), (0.5760869565217391, 50, 10, 10, 8), (0.5760869565217391, 20, 10, 5, 8), (0.5760869565217391, 70, 10, 5, 2), (0.5760869565217391, 102, 10, 5, 2), (0.5760869565217391, 50, 10, None, 8), (0.5760869565217391, 70, 100, 100, 16), (0.5760869565217391, 70, 100, 100, 8), (0.5760869565217391, 20, 100, 10, 16), (0.5760869565217391, 90, 100, 10, 8), (0.5760869565217391, 70, 100, 10, 2), (0.5760869565217391, 70, 100, None, 16), (0.5760869565217391, 70, 100, None, 8), (0.5652173913043478, 70, 500, 100, 4), (0.5652173913043478, 70, 500, 10, 8), (0.5652173913043478, 70, 500, 10, 4), (0.5652173913043478, 50, 500, 5, 16), (0.5652173913043478, 50, 500, 5, 8), (0.5652173913043478, 70, 500, None, 4), (0.5652173913043478, 102, 50, 100, 16), (0.5652173913043478, 70, 50, 100, 8), (0.5652173913043478, 90, 50, 100, 4), (0.5652173913043478, 70, 50, 100, 2), (0.5652173913043478, 102, 50, 10, 8), (0.5652173913043478, 90, 50, 10, 4), (0.5652173913043478, 20, 50, 5, 16), (0.5652173913043478, 102, 50, None, 16), (0.5652173913043478, 70, 50, None, 8), (0.5652173913043478, 90, 50, None, 4), (0.5652173913043478, 70, 50, None, 2), (0.5652173913043478, 50, 10, 100, 16), (0.5652173913043478, 20, 10, 100, 8), (0.5652173913043478, 90, 10, 100, 4), (0.5652173913043478, 20, 10, 10, 8), (0.5652173913043478, 102, 10, 10, 8), (0.5652173913043478, 20, 10, 10, 4), (0.5652173913043478, 20, 10, 5, 16), (0.5652173913043478, 102, 10, 5, 4), (0.5652173913043478, 50, 10, 5, 2), (0.5652173913043478, 50, 10, None, 16), (0.5652173913043478, 20, 10, None, 8), (0.5652173913043478, 90, 10, None, 4), (0.5652173913043478, 50, 100, 10, 16), (0.5652173913043478, 70, 100, 10, 8), (0.5652173913043478, 20, 100, 5, 16), (0.5652173913043478, 50, 100, 5, 16), (0.5543478260869565, 102, 500, 100, 8), (0.5543478260869565, 90, 500, 100, 2), (0.5543478260869565, 102, 500, 10, 16), (0.5543478260869565, 102, 500, 10, 8), (0.5543478260869565, 50, 500, 5, 4), (0.5543478260869565, 50, 500, 5, 2), (0.5543478260869565, 102, 500, None, 8), (0.5543478260869565, 90, 500, None, 2), (0.5543478260869565, 90, 50, 100, 16), (0.5543478260869565, 90, 50, 100, 2), (0.5543478260869565, 90, 50, 10, 16), (0.5543478260869565, 102, 50, 10, 16), (0.5543478260869565, 70, 50, 10, 8), (0.5543478260869565, 70, 50, 10, 2), (0.5543478260869565, 90, 50, 10, 2), (0.5543478260869565, 90, 50, None, 16), (0.5543478260869565, 90, 50, None, 2), (0.5543478260869565, 102, 10, 100, 16), (0.5543478260869565, 102, 10, 100, 8), (0.5543478260869565, 70, 10, 10, 16), (0.5543478260869565, 90, 10, 10, 2), (0.5543478260869565, 102, 10, 5, 8), (0.5543478260869565, 70, 10, 5, 4), (0.5543478260869565, 20, 10, 5, 2), (0.5543478260869565, 102, 10, None, 16), (0.5543478260869565, 102, 10, None, 8), (0.5543478260869565, 102, 100, 100, 16), (0.5543478260869565, 90, 100, 100, 8), (0.5543478260869565, 90, 100, 100, 2), (0.5543478260869565, 102, 100, 10, 16), (0.5543478260869565, 50, 100, 5, 8), (0.5543478260869565, 102, 100, None, 16), (0.5543478260869565, 90, 100, None, 8), (0.5543478260869565, 90, 100, None, 2), (0.5434782608695652, 90, 500, 100, 8), (0.5434782608695652, 102, 500, 100, 2), (0.5434782608695652, 90, 500, 10, 2), (0.5434782608695652, 90, 500, None, 8), (0.5434782608695652, 102, 500, None, 2), (0.5434782608695652, 90, 50, 100, 8), (0.5434782608695652, 90, 50, None, 8), (0.5434782608695652, 70, 10, 100, 16), (0.5434782608695652, 70, 10, 100, 2), (0.5434782608695652, 90, 10, 100, 2), (0.5434782608695652, 50, 10, 10, 16), (0.5434782608695652, 102, 10, 10, 16), (0.5434782608695652, 70, 10, 10, 8), (0.5434782608695652, 50, 10, 10, 4), (0.5434782608695652, 102, 10, 10, 2), (0.5434782608695652, 50, 10, 5, 16), (0.5434782608695652, 102, 10, 5, 16), (0.5434782608695652, 50, 10, 5, 4), (0.5434782608695652, 90, 10, 5, 2), (0.5434782608695652, 70, 10, None, 16), (0.5434782608695652, 70, 10, None, 2), (0.5434782608695652, 90, 10, None, 2), (0.5434782608695652, 90, 100, 100, 16), (0.5434782608695652, 90, 100, 100, 4), (0.5434782608695652, 102, 100, 100, 4), (0.5434782608695652, 102, 100, 10, 8), (0.5434782608695652, 90, 100, 10, 2), (0.5434782608695652, 90, 100, None, 16), (0.5434782608695652, 90, 100, None, 4), (0.5434782608695652, 102, 100, None, 4), (0.532608695652174, 90, 500, 10, 8), (0.532608695652174, 90, 500, 10, 4), (0.532608695652174, 102, 500, 10, 4), (0.532608695652174, 102, 500, 10, 2), (0.532608695652174, 70, 500, 5, 2), (0.532608695652174, 102, 50, 100, 8), (0.532608695652174, 50, 50, 5, 16), (0.532608695652174, 50, 50, 5, 8), (0.532608695652174, 50, 50, 5, 4), (0.532608695652174, 102, 50, None, 8), (0.532608695652174, 70, 10, 100, 8), (0.532608695652174, 50, 10, 100, 4), (0.532608695652174, 102, 10, 10, 4), (0.532608695652174, 70, 10, 10, 2), (0.532608695652174, 70, 10, 5, 16), (0.532608695652174, 70, 10, None, 8), (0.532608695652174, 50, 10, None, 4), (0.532608695652174, 90, 100, 10, 16), (0.532608695652174, 90, 100, 10, 4), (0.532608695652174, 102, 100, 10, 4), (0.532608695652174, 50, 100, 5, 2), (0.532608695652174, 70, 100, 5, 2), (0.5217391304347826, 102, 500, 100, 16), (0.5217391304347826, 90, 500, 100, 4), (0.5217391304347826, 102, 500, 100, 4), (0.5217391304347826, 70, 500, 5, 16), (0.5217391304347826, 70, 500, 5, 8), (0.5217391304347826, 70, 500, 5, 4), (0.5217391304347826, 102, 500, None, 16), (0.5217391304347826, 90, 500, None, 4), (0.5217391304347826, 102, 500, None, 4), (0.5217391304347826, 90, 50, 10, 8), (0.5217391304347826, 102, 50, 10, 2), (0.5217391304347826, 90, 50, 5, 8), (0.5217391304347826, 50, 50, 5, 2), (0.5217391304347826, 102, 50, 5, 2), (0.5217391304347826, 90, 10, 10, 16), (0.5217391304347826, 20, 10, 10, 2), (0.5217391304347826, 50, 10, 10, 2), (0.5217391304347826, 90, 10, 5, 4), (0.5217391304347826, 102, 100, 100, 8), (0.5217391304347826, 102, 100, 10, 2), (0.5217391304347826, 70, 100, 5, 16), (0.5217391304347826, 50, 100, 5, 4), (0.5217391304347826, 90, 100, 5, 2), (0.5217391304347826, 102, 100, None, 8), (0.5108695652173914, 90, 500, 100, 16), (0.5108695652173914, 90, 500, 10, 16), (0.5108695652173914, 90, 500, None, 16), (0.5108695652173914, 70, 50, 5, 16), (0.5108695652173914, 102, 50, 5, 4), (0.5108695652173914, 90, 50, 5, 2), (0.5108695652173914, 102, 10, 100, 4), (0.5108695652173914, 50, 10, 100, 2), (0.5108695652173914, 102, 10, None, 4), (0.5108695652173914, 50, 10, None, 2), (0.5108695652173914, 102, 100, 100, 2), (0.5108695652173914, 70, 100, 5, 8), (0.5108695652173914, 70, 100, 5, 4), (0.5108695652173914, 102, 100, None, 2), (0.5, 102, 50, 100, 2), (0.5, 102, 50, 5, 8), (0.5, 70, 50, 5, 4), (0.5, 90, 50, 5, 4), (0.5, 102, 50, None, 2), (0.5, 90, 10, 100, 16), (0.5, 102, 10, 100, 2), (0.5, 90, 10, 5, 16), (0.5, 90, 10, 5, 8), (0.5, 90, 10, None, 16), (0.5, 102, 10, None, 2), (0.5, 90, 100, 5, 4), (0.4891304347826087, 90, 500, 5, 16), (0.4891304347826087, 90, 500, 5, 4), (0.4891304347826087, 90, 500, 5, 2), (0.4891304347826087, 70, 50, 5, 2), (0.4891304347826087, 90, 10, 100, 8), (0.4891304347826087, 90, 10, 10, 4), (0.4891304347826087, 90, 10, None, 8), (0.4891304347826087, 90, 100, 5, 8), (0.4891304347826087, 102, 100, 5, 2), (0.4782608695652174, 102, 500, 5, 16), (0.4782608695652174, 90, 500, 5, 8), (0.4782608695652174, 102, 500, 5, 8), (0.4782608695652174, 102, 500, 5, 4), (0.4782608695652174, 102, 500, 5, 2), (0.4782608695652174, 90, 50, 5, 16), (0.4782608695652174, 70, 50, 5, 8), (0.4782608695652174, 102, 100, 5, 16), (0.4782608695652174, 102, 100, 5, 8), (0.4782608695652174, 102, 100, 5, 4), (0.4673913043478261, 90, 100, 5, 16), (0.45652173913043476, 102, 50, 5, 16), (0.43478260869565216, 70, 10, 5, 8)]\n"
     ]
    }
   ],
   "source": [
    "NumComponents = [10,20,50,70,90,102]\n",
    "results = []\n",
    "for n_est in n_estimators:\n",
    "    for m_dep in max_depth:\n",
    "        for min_sam in min_sample_split:\n",
    "            for num in NumComponents:\n",
    "                pca = PCA(n_components=num)\n",
    "                X_train1 = pca.fit_transform(X_train)\n",
    "                X_test1 = pca.transform(X_test)\n",
    "                rf = RandomForestClassifier(random_state=0,n_estimators=n_est,max_depth=m_dep,min_samples_split=min_sam)\n",
    "                rf.fit(X_train1,y_train)\n",
    "                y_pred_pca = rf.predict(X_test1)\n",
    "                tup = (accuracy_score(y_test, y_pred_pca),num,n_est,m_dep,min_sam)\n",
    "                results.append(tup)\n",
    "                #print(f'Accuracy with {num} components, n_estimators = {n_est}, max_depth = {m_dep}, min_samples_split = {min_sam}: {accuracy_score(y_test, y_pred_pca)}\\n')\n",
    "\n",
    "results.sort(key = lambda x: x[0],reverse = True)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.6847826086956522, 10, 500, 10, 2),\n",
       " (0.6195652173913043, 20, 500, 10, 2),\n",
       " (0.5869565217391305, 50, 500, 10, 2),\n",
       " (0.5869565217391305, 70, 500, 10, 2),\n",
       " (0.5434782608695652, 90, 500, 10, 2),\n",
       " (0.532608695652174, 102, 500, 10, 2)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item for item in results if (item[2] == 500) and (item[3] == 10) and (item[4] == 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = Pipeline([\n",
    "  ('feature_selection', SelectFromModel(LinearSVC(penalty=\"l1\", dual=False, C=0.01))),\n",
    "  ('classification', RandomForestClassifier())\n",
    "])\n",
    "model = clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 3, 1, 1, 1, 1, 2, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 1, 1, 0, 2,\n",
       "       1, 0, 2, 0, 2, 0, 1, 0, 0, 1, 2, 2, 0, 1, 0, 1, 0, 2, 1, 1, 2, 0,\n",
       "       1, 0, 3, 1, 3, 1, 2, 2, 2, 0, 2, 1, 0, 1, 1, 2, 1, 0, 0, 0, 0, 0,\n",
       "       2, 2, 0, 1, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0,\n",
       "       0, 0, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_classification_metrics(y_test, y_pred):\n",
    "    print(classification_report(y_test, y_pred))\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    cmd = ConfusionMatrixDisplay(cm, display_labels=[\"nonevent\", \"II\", \"Ib\", \"Ia\"])\n",
    "    _ = cmd.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.72      0.65        36\n",
      "           1       0.43      0.43      0.43        28\n",
      "           2       0.41      0.29      0.34        24\n",
      "           3       0.00      0.00      0.00         4\n",
      "\n",
      "    accuracy                           0.49        92\n",
      "   macro avg       0.36      0.36      0.36        92\n",
      "weighted avg       0.47      0.49      0.47        92\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVsAAAEGCAYAAADL3zbEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAg9UlEQVR4nO3deXxU1d3H8c8vIUAImxBEQCioiPuCEWttKbgrVkXbauvTWquCVh9rS1ut7VOtvuqrPC5ttdo+uNSlLm1dKm4sVZC6IiAiKLixSVAIa4QAIfk9f9wbHShkJsm9c2cm3/frdV8zc+fMub+bwC9nzr3nHHN3REQkXkVJByAi0hoo2YqIZIGSrYhIFijZiohkgZKtiEgWtEk6gHxS3q3Y+/ctSTqMyL27sDzpEKSpqjcmHUFsqllT5e49WlLHCcPLfNXqurTlZs7ZPNHdT2zJsTKlZNsE/fuWMH1i36TDiNyx3/5+0iFIExVPnZV0CLH5lz+yuKV1VK2u47WJu6ctV9Lrg6y1NJRsRaQAOXVen3QQ21CyFZGC40A9uTVgS8lWRApSPWrZiojEynFqc6wbQbd+iUjBcaAOT7ulY2Z9zWyKmb1jZvPM7Ifh/mvMbJmZzQ63k9PVpZatiBSkiPpstwJj3H2WmXUCZprZ5PC937n7jZlWpGQrIgXHgboIZjR09+XA8vB5tZm9A/RpTl3qRhCRglSfwQaUm9mMlG3Uzuozs/7AocBr4a5LzWyOmd1tZruki0ctWxEpOJ5hnyxQ5e4V6QqZWUfgUeByd19vZn8CriNoRF8H3AQ0OjpIyVZECo471EZ0m62ZlRAk2gfc/bGgfv8k5f07gKfS1aNkKyIFyKjDWl6LmQF3Ae+4+80p+3uF/bkAI4G56epSshWRguNAfTQt26OA7wBvmdnscN9VwLfM7JDwUIuA0ekqUrIVkYIURcvW3V+EHVb0TFPrUrIVkYITDGpoebKNkpKtiBQcB2o9t+5sVbIVkYLjGHU5NoxAyVZEClK9qxtBRCRW6rMVEckKo059tiIi8QpWalCyFRGJlbuxxYuTDmMbSrY5YMWyEm74YT/WrCjBipyT/2sVIy+oAuCJu8oZ/5dyito4Rxyzngv+Z3ma2nLXX//wD2pq2lBXX0RdvXHJL09NOqTIFOq5VQxbz0XXVVJc5Dz7UDf+/seeSYeUsXr12eYeM7scGOfuG5M4fnEbZ9SvKhl4UA0bPy3i0hP3ZvDQatasLOHliV3403MLaNvOWVuV/7+uMb85ifXV7ZMOIxaFdm5FRc4l1y/j52fvQdXyEm595j1endiFJe/l/jkGF8hyqxsht6JJzuVAh6QO3r3nVgYeVANAh4719N1rM1XLS3jqvu6cdekntG0XDPLuWr41qRClFRp06EYqF7Xl4yXt2FpbxNQnunLkCeuSDitDwQWydFs2xXY0M+sfrttzR7h2zyQzKzWzQ8zs1XDS3ccbJt01s6lmNtbMppvZu2b2lXB/sZndYGavh58ZHe7/W+q6P2Z2j5md2Uj5YeExHjGz+Wb2gAUuA3oDU8xsSlw/j0x9vLQtH8wtZZ/BG1n2QXvmvtaRy0YM5Cdn7MWC2aVJh9ci7jD2yonc/pvxjDh6QdLhRKoQz637brWsrGz72euq5SWU96pNMKLMNVwgS7dlU9zfSwcC33L3C83s78CZwM+A/3b3F8zsWuBqgpYlQBt3HxIm0auBY4HzgXXufriZtQNeMrNJwMPAWcAzZtYWOAa4uJHyEMyyvj9QCbwEHOXut5jZj4Hh7l4V88+jUTUbirjugv5cdO0yyjrVU1cHn64r5g9PvceC2R34zej+3PvqO1hudUVl7PJrRrBqbQe6dq5h7M8nsqSyC2/N3y3psCJRiOe2o39nEaw0kzV1OTaoIe7UvtDdZ4fPZwJ7Al3d/YVw373A0JTyj6WU7R8+Px74bji92WtAd4Ik/ixwdJhQTwKmuXtNI+UBprv7R+5eD8xOOcZOmdmohiUzVq6qy/jEm2prLVx3QX+OPmMNXz45+KpW3quWo05ehxnsc+hGiopg3ercusLaFKvWBj01a9eX8tKML7DPnisTjig6hXhuVctL6NF7y2evy3vVsurjkgQjypxj1HqbtFs2xZ1sN6c8rwO6Zli+js9b3UbQEj4k3Aa4+yR33wRMBU4gaOE+3Fj5ncST9qft7uPcvcLdK3p0jyfRucPNY/rRd+Bmzhz9+X/SL524jtkvdgTgow/aUbvF6NItvoQfp/btailtX/vZ88MOXMaipWmXbcoLhXpuC2Z3oM+ALfTsu5k2JfUMO20tr07qknRYGWm4QJZuy6ZsX95eB6wxs6+4+78JJuV9Ic1nJgIXm9nz7l5rZnsDy9x9A0GCvQCoAL7XWPk0x6gGOgGJdCPMm17Gc490Y8C+NVx87CAAzvt5JSecvZqbf9yXUcMHUVLi/PQPS/K2C2GXLpu45kfPAVBc7Dz/0h68Pmf3hKOKRqGeW32dcdsv+nD9gx9SVAyTHu7G4ndz/04ECCeiybFuhCTuJToX+LOZdQA+BM5LU/5Ogq/7s8IlKlYCp4fvTQLuA8a7+5YMyu/MOOBZM1vu7sObcjJROOCIDUysnL3D967445LsBhOT5Ss6MfrnpycdRiwK+dxef74zrz/fOekwmqXVjCBz90XAASmvb0x5+4s7KD8s5XkVYX9q2L96Vbht/5lagj7Z1H07Kz813BrKXZry/Fbg1sbPSETyhTuaG0FEJG7BBbLcupisZCsiBSnXRpAp2YpIwXFMk4eLiGSDWrYiIjFzoF4XyERE4mZaFkdEJG7BUua6G0FEJFbupm4EEZFs0KAGEZGYBfPZqs9WRCRmWspcRCR2wa1fatmKiMRKcyOIiGRJrk2xmFvRiIhEIJhi0dJu6ZhZXzObEi5eO8/Mfhju72Zmk83svfAx7dIcSrYiUpDq3dJuGdgKjHH3fQnm4b7EzPYDrgSec/eBwHPh60Yp2YpIwQlm/SpKu6Wtx325u88Kn1cD7wB9gNMIFqwlfDw9XV3qsxWRghMM1422LWlm/YFDCVbt7unuyyFIyGa2a7rPK9mKSAHKeLhuuZnNSHk9zt3H/UdtZh2BR4HL3X29NWPlVSVbESlIGY4gq3L3isYKmFkJQaJ9wN0fC3d/Yma9wlZtL2BFugOpz1ZECk6EdyMYcBfwjrvfnPLWeIKVwgkfn0hXl1q2TTDvkx4cdNMPkg4jcpuO96RDiM0ej1UnHUIsivv0TjqE+HwUTTURzfp1FPAd4C0zmx3uuwr4LfB3MzsfWAJ8I11FSrYiUnCiWoPM3V+EnfZHHNOUupRsRaTgOLBVE9GIiMRPk4eLiMQt8xFiWaNkKyIFR5OHi4hkiVq2IiIx0+ThIiJZ4Bhb63WBTEQkduqzFRGJm6sbQUQkduqzFRHJEiVbEZGYOUadLpCJiMRPF8hERGLmukAmIpIdrmQrIhI3TUQjIpIVatmKiMTMHerqlWxFRGKnuxFERGLmqBtBRCQLdIFMRCQr3JOOYFtKtjng1ydM4at7LGL1xlLOuPdsADq338QNp0ymd+dqKtd34idPHk/15nYJR9p01x81heG7L2bVplJOeeIsAH5W8QpH913MlroillZ35sqXhlO9Jf/OLVVRUT233DyBVatKufq64UmHE4nynjWMuWYOu3TfTL0bEx7vy/iH+ycdVsZyrRshtwYPJ8DMPg0f+5vZ3CRiGD93EBc/eso2+84f8gavLenD1+7+Nq8t6cP5Q2YlEVqLPfb+IM6fPGKbfS9V7s6If36TU8d/k4XruzL6wDcSii46p39tAUuXdk46jEjVbTXu/P0+XPTNoYw570hO+fpi+g6oTjqsjAR3IxSl3bKp1SfbXDBzWW/Wbdq2ZTd8z4WMnzcIgPHzBnH0XguTCK3FZnzSm3XbtVpfquxLXbjM9Jsre7Jbh0+TCC0y5d03cnjFMiZM3ivpUCK1ZlV7PljQBYCajW1Yuqgj3XtsTjiqzLmn37JJyTZHdetQQ9WGMgCqNpTRrUNNwhHF48yB85m2rF/SYbTI6AtmcNc9h+I5dl9nlHbttZE9Bq1nwbwuSYeSMXdLu2WTkm0aZjbKzGaY2Yy6jRuSDqegXHTQTOrqjfEfDkw6lGYbUvERa9e15/0PuicdSmzal27lF2Pf4I6b96VmQ0nS4WTESZ9os51sdYEsDXcfB4wDKN2tb9a+eKzeWEp52QaqNpRRXraB1RtLs3XorBi55wKG776EcyeeAjl283lT7L/fSr445COGHFZJSds6OnSo5Wc/fon/vfmopEOLRHFxPVeNfYMpE3rz8pTdkg6nSXLsZgQl21w19YP+nLr/Au6ePphT91/AlA8GJB1SZL7SZwkXHjibc549lU11+dFS2pm/3Hcof7nvUAAOOuATzhz5dsEkWnB++D9vsXRRGf98MM/+/Tk5162jZJsDxo6YTMXulXQt3cTkUfdx+8uHc9f0wdx4yiRGHjCfj9d3ZMxTxycdZrPcPPRfDNmtkl3ab2LaN+7nltkVjD7wDdoW13HPCU8BMHtlT65+ZWjCkcr29jt4DceMqGThe5249YEXAbj3tr2Z8fKuCUeWmVy79UvJNgdc8fRxO9x/4SOnZjmS6P142rH/se+R9/ZNIJL4zZnbkzlzeyYdRmTefrMbIw4/Kekwmi1vBjWY2a000u3h7pfFElGWuXvH8HERcECy0YhIFPJtboQZWYtCRCRKDuRLsnX3e1Nfm1mZu+veJxHJC1F1I5jZ3cApwAp3PyDcdw1wIbAyLHaVuz/TWD1p77M1syPN7G3gnfD1wWZ2ewtiFxGJmeH16bcM3QOcuIP9v3P3Q8Kt0UQLmQ1q+D1wArAKwN3fBHTpWERym2ewZVKN+zRgdUvDyWgEmbsv3W5XXUsPLCISG894uG55wwjRcBvVhKNcamZzzOxuM9slXeFMku1SM/sS4GbW1sx+QtilICKSszJr2Va5e0XKNi7D2v8E7AkcAiwHbkr3gUyS7UXAJUAfYFlY+SUZBiQikhDLYGsed//E3evcvR64AxiS7jNpBzW4exVwTrOjEhFJQn18VZtZL3dfHr4cCaSdCzttsjWzPYA/AF8kaHi/AvzI3T9sQawiIvGJ8D5bM3sIGEbQv/sRcDUwzMwOCY+0CBidrp5Mhus+CNxGkL0BzgYeAo5oatAiItkS1X227v6tHey+q6n1ZNJna+5+v7tvDbe/knuzl4mIbCuiW7+i0tjcCN3Cp1PM7ErgYYLwzgKezkJsIiLNly/DdYGZBMm1IeLUPgkHrosrKBGRlrIc+/7d2NwIeTZbsIhIyA3ycfJwMzsA2A9o37DP3e+LKygRkRbLl5ZtAzO7muC2h/2AZ4CTgBcBJVsRyV05lmwzuRvh68AxwMfufh5wMNAu1qhERFoqX+5GSFHj7vVmttXMOgMrgD1ijktEpPnyafLwFDPMrCvB+N+ZwKfA9DiDEhFpqby5G6GBu/8gfPpnM5sAdHb3OfGGJSLSQvmSbM1scGPvufuseEISEWm5fGrZNjY/owNHRxxLzmuzyek2vzbpMCK3ep+SpEOITfWAsqRDiEXnBYuTDiH35UufrbsPz2YgIiKRSeBug3QyGtQgIpJ3lGxFROJnMU4e3hxKtiJSmHKsZZt2BJkF/svMfhW+7mdmadfbERFJinlmWzZlMlz3duBIoGG28mqClRtERHKXW/otizLpRjjC3Qeb2RsA7r7GzNrGHJeISMvkWDdCJsm21syKCUM3sx7Eum6liEjL5dqghky6EW4BHgd2NbPfEEyveH2sUYmItIQHdyOk27Ipk7kRHjCzmQTTLBpwuru/E3tkIiItkWMt20wmD+8HbASeTN3n7kviDExEpEXyLdkSrKTbsPBje2AAsADYP8a4RERaJNf6bDPpRjgw9XU4G9jonRQXEZEdaPIIMnefZWaHxxGMiEhk8q1la2Y/TnlZBAwGVsYWkYhIS3l+zo3QKeX5VoI+3EfjCUdEJCL51LINBzN0dPefZikeEZEWM/LoApmZtXH3rY0tjyMikrPyJdkSrKA7GJhtZuOBfwAbGt5098dijk1EpHkSmNUrnUz6bLsBqwjWHGu439YBJVsRyV15dIFs1/BOhLl8nmQb5NjfDBGRbeVay7axiWiKgY7h1inlecMmIpK7PIMtA2Z2t5mtMLO5Kfu6mdlkM3svfNwlXT2NtWyXu/u1mYUjUepYupmfnvtvBvRegwNj7xnK2x/2TDqsZvn1CVP46h6LWL2xlDPuPRuAzu03ccMpk+nduZrK9Z34yZPHU725XcKRNl+/Xddy7bnPffa6d/l67nymgr+/cGAjn8oPP7r+XYYMW8PaVSVc/LU8ulYe7eq69wB/BO5L2Xcl8Jy7/9bMrgxfX9FYJY21bHNr0fWImNmn4eMwM3sq6Xh25NKzX2X63N357q++wfm/PoMly7smHVKzjZ87iIsfPWWbfecPeYPXlvTha3d/m9eW9OH8IbMSii4aS1Z05Xs3nMn3bjiT7984kk1b2vDCnP5JhxWJyY/15JcX5Oc0KFEti+Pu04DV2+0+Dbg3fH4vcHq6ehpLtsdkFopEqUP7LRy893KefnEQAFvrivm0Jn9bfTOX9Wbdpm3jH77nQsbPC85v/LxBHL3XwiRCi0XF3pUsq+rMJ2s6pS+cB+bO6EL1ujxdFzazboRyM5uRso3KsPae7r4cIHzcNd0HdvpTdPftM3kh6mxmjwODgGnAD9w90WuYvXtUs7a6lCvPm8aeu6/m3cXdufXhI9m0pSTJsCLVrUMNVRvKAKjaUEa3DjUJRxSdYwa/z79m7Zl0GELGw3Wr3L0i5lCAzFZqKGRDgDHAgcCewBnbFzCzUQ1/9Wq3bNj+7cgVF9Wzd78qnpi6LxdeN5KazSV8+6Q3Yz+utFyb4jq+fMBinp+9R9KhSCat2pb16X5iZr0AwscV6T7Q2pPtdHf/0N3rgIeAL29fwN3HuXuFu1eUtC2LPaCVa8pYuaaMdxYG30pemDWAgf1WxX7cbFq9sZTysuAPV3nZBlZvLE04omh8cd+lvPtROWuqOyQdSqtnGW4tMB44N3x+LvBEug+09mS7/d+2xO/MW72+AyvWlNG351oADttnGYvz+ALZjkz9oD+n7r8AgFP3X8CUDwYkHFE0jjvsfSbP2ivpMKRBdLd+PQS8Agwys4/M7Hzgt8BxZvYecFz4ulF52vMdmSFmNgBYDJwFjEs4HgBueehL/PKCqbRpU8fylZ357T1Dkw6p2caOmEzF7pV0Ld3E5FH3cfvLh3PX9MHceMokRh4wn4/Xd2TMU8cnHWaLtSvZyuGDlvG/f8vf39WOXHHTfA4aso7Ou2zl/hemc/+t/Zj0yG5Jh5WRqAY1uPu3dvJWk24iaO3J9hWCv0gHElwgezzZcALvL+3O6N+cnnQYkbji6eN2uP/CR07NciTx2lzbhpOvOjd9wTwzdsw+SYfQfIl/T91Wq0u27t4xfJwKTE00GBGJR55OHi4ikn/UshURiV+uTUSjZCsihUnJVkQkfmrZiojEzcmrycNFRPJSXi34KCKS15RsRUTiZ55b2VbJVkQKT7QrNURCyVZECpL6bEVEskDDdUVEskEtWxGRmDVhQcdsUbIVkcKkZCsiEi8NahARyRKrz61sq2QrIoVH99mKiGSHbv0SEckGtWxFROKnC2QiInFzQBPR5K+ijbWUzV6WdBiRK51WnXQI0kT11fqdpaM+WxGRmOk+WxGRbHBXN4KISDaoZSsikg1KtiIi8VPLVkQkbg7U5Va2VbIVkYKklq2ISDbobgQRkfipZSsiErcIp1g0s0VANVAHbHX3iubUo2QrIgXHAIv2Atlwd69qSQVKtiJSkCzH+myLkg5ARCRynuEG5WY2I2UbtZPaJpnZzJ28nxG1bEWkAGU8N0JVBn2wR7l7pZntCkw2s/nuPq2pEallKyIFyTz9lgl3rwwfVwCPA0OaE4+SrYgUpoaZvxrb0jCzMjPr1PAcOB6Y25xw1I0gIoXHI7sboSfwuJlBkC8fdPcJzalIyVZEClMEudbdPwQObnlNSrYiUqBy7dYvJVsRKUxKtiIiMXNACz6KiMTLcHUjSOPKe9Yw5po57NJ9M/VuTHi8L+Mf7p90WJH40fXvMmTYGtauKuHirw1OOpzIFOp5AVQMW89F11VSXOQ8+1A3/v7HnkmHlLn63Gratrr7bM3s06RjaEzdVuPO3+/DRd8cypjzjuSUry+m74DqpMOKxOTHevLLC/ZPOozIFep5FRU5l1y/jF+eM4ALhw1i+Glr6TdwU9JhZaahGyHdlkWtLtnmujWr2vPBgi4A1Gxsw9JFHeneY3PCUUVj7owuVK8rvC9ThXpegw7dSOWitny8pB1ba4uY+kRXjjxhXdJhZczc027Z1GqTrZl1NLPnzGyWmb1lZqclHdP2du21kT0GrWfBvC5JhyKtUPfdallZ2faz11XLSyjvVZtgRE0UwQiyKBXen+PMbQJGuvt6MysHXjWz8e650avevnQrvxj7BnfcvC81G0qSDkdaoWDQ1LZy439HJrKfTNNpzcnWgOvNbChB700fgqF5H29TKJhSbRRA++JOWQmsuLieq8a+wZQJvXl5ym5ZOabI9qqWl9Cj95bPXpf3qmXVx3nyhz8HV9dttd0IwDlAD+Awdz8E+ARov30hdx/n7hXuXtG2qDQLYTk//J+3WLqojH8+OCALxxPZsQWzO9BnwBZ69t1Mm5J6hp22llcn5U+XVq712bbmlm0XYIW715rZcOALSQcEsN/BazhmRCUL3+vErQ+8CMC9t+3NjJd3TTiylrvipvkcNGQdnXfZyv0vTOf+W/sx6ZH8b7kX6nnV1xm3/aIP1z/4IUXFMOnhbix+9z/aI7lL3Qg54wHgSTObAcwG5icbTuDtN7sx4vCTkg4jFmPH7JN0CLEo1PMCeP35zrz+fOekw2g6B+qVbBPl7h3DxyrgyITDEZFY6AKZiEh2KNmKiMTMgbrcGq6rZCsiBcjBlWxFROKnbgQRkZjpbgQRkSxRy1ZEJAuUbEVEYuYOdXVJR7ENJVsRKUxq2YqIZIGSrYhI3Fx3I4iIxM7BNahBRCQLNFxXRCRm7jm3lLmSrYgUJl0gExGJn6tlKyISN00eLiISP01EIyISPwc8x4brtualzEWkUHk4eXi6LQNmdqKZLTCz983syuaGpJatiBQkj6AbwcyKgduA44CPgNfNbLy7v93UutSyFZHCFE3Ldgjwvrt/6O5bgIeB05oTjnmOXbHLZWa2ElicxUOWA1VZPF626LzyTzbP7Qvu3qMlFZjZBIKY02kPbEp5Pc7dx6XU83XgRHe/IHz9HeAId7+0qTGpG6EJWvoPoKnMbIa7V2TzmNmg88o/+XZu7n5iRFXZjqpvTkXqRhAR2bmPgL4pr3cHKptTkZKtiMjOvQ4MNLMBZtYWOBsY35yK1I2Q28alL5KXdF75p5DPbafcfauZXQpMBIqBu919XnPq0gUyEZEsUDeCiEgWKNmKiGSBkm0rYmaXm1mHBI//afjY38zmJhVHlFLOaZiZPZV0PFFpOC+JjpJt63I5kFiyFWnNlGxbKGylvWNmd5jZPDObZGalZnaImb1qZnPM7HEz2yUsP9XMxprZdDN718y+Eu4vNrMbzOz18DOjw/1/M7OTU453j5md2Uj5YeExHjGz+Wb2gAUuA3oDU8xsSvZ/Uq1C5/B3/baZ/dnM8v7/l5l1NLPnzGyWmb1lZs0aqipKtlEZCNzm7vsDa4EzgfuAK9z9IOAt4OqU8m3cfQhBS7Nh//nAOnc/HDgcuNDMBhCMxT4LILzP7xjgmUbKAxwa1r0fsAdwlLvfQnAz9nB3Hx71D0CAYBz9GOBAYE/gjGTDicQmYKS7DwaGAzeZ2Y5GVUkaSrbRWOjus8PnMwn+o3V19xfCffcCQ1PKP5ZStn/4/Hjgu2Y2G3gN6E6QxJ8FjjazdsBJwDR3r2mkPMB0d//Ig7WcZ6ccQ+I1PZywpA54CPhy0gFFwIDrzWwO8C+gD9Az2ZDykwY1RGNzyvM6oGuG5ev4/HdgwH+7+8TtC5vZVOAEghbuQ42VN7NhO4hHv+fs2P6m9UK4if0coAdwmLvXmtkigslbpInUso3HOmBNQ38s8B3ghUbKQzBC5WIzKwEws73NrCx872HgPOArYbl05XemGujUpDORphgSDussIvjD+GLSAUWgC7AiTLTDgS8kHVC+UosnPucCfw5vtfqQIFk25k6Cr/uzwj6xlcDp4XuTCPqAx4dzaqYrvzPjgGfNbLn6bWPxCvBbgj7bacDjyYYTiQeAJ81sBkGX1Pxkw8lfGq4rIpIF6kYQEckCJVsRkSxQshURyQIlWxGRLFCyFRHJAiVbiZSZ1ZnZbDOba2b/aMksY+E8EF8Pn99pZvs1UnaYmX2pGcdYZGb/sQrrzvZvV6ZJM2OZ2TVm9pOmxiiFQclWolbj7oe4+wHAFuCi1DfNrLg5lbr7Be7+diNFhgFNTrYi2aJkK3H6N7BX2OqcYmYPAm81MmOZmdkfw1mzngZ2bagonMmsInx+YjgL1ZvhjFT9CZL6j8JW9VfMrIeZPRoe43UzOyr8bPdwZrY3zOz/2PFS1dsws3+a2UwLZnUbtd17N4WxPGdmPcJ9e5rZhPAz/zazfSL5aUpe0wgyiYWZtSGYOGdCuGsIcIC7LwwT1jp3PzycYOclM5tEMFvZIIIRWD2Bt4G7t6u3B3AHMDSsq5u7rzazPwOfuvuNYbkHgd+5+4tm1o9gePO+BLOsveju15rZCGCb5LkT3w+PUQq8bmaPuvsqoAyY5e5jzOxXYd2XEozUu8jd3zOzI4DbgaOb8WOUAqJkK1ErDWcig6BlexfB1/vp7r4w3H88cFBDfyzB+PuBBDOjPRTOmlVpZs/voP4vEsx8thDA3VfvJI5jgf1SZgPsbGadwmOcEX72aTNbk8E5XWZmI8PnfcNYVwH1wN/C/X8FHjOzjuH5/iPl2O0yOIYUOCVbiVqNux+SuiNMOhtSd7HjGctOJv1MWZZBGQi6yI4Mp6PcPpaMx6iHs6gdG9a1MZyBbWezXnl43LXb/wxE1GcrSdjZjGXTgLPDPt1eBJNVb+8V4KsNE6WbWbdw//Yzmk0i+EpPWO6Q8Ok0gmkDMbOTgF3SxNoFWBMm2n0IWtYNioCG1vm3Cbon1gMLzewb4THMzA5OcwxpBZRsJQl3EvTHzrJg4cf/I/iW9TjwHsHKFn9iB9NSuvtKgn7Wx8zsTT7/Gv8kMLLhAhlwGVARXoB7m8/vivg1MNTMZhF0ZyxJE+sEoI0Fk2dfB7ya8t4GYH8zm0nQJ3ttuP8c4PwwvnmAlpIRzfolIpINatmKiGSBkq2ISBYo2YqIZIGSrYhIFijZiohkgZKtiEgWKNmKiGTB/wO46KYJvQZ6XgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print_classification_metrics(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.6913365420214734\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.6913365420214734\n",
      "\n",
      "Generation 3 - Current best internal CV score: 0.6940762680488708\n",
      "\n",
      "Generation 4 - Current best internal CV score: 0.6968530174009626\n",
      "\n",
      "Generation 5 - Current best internal CV score: 0.6968530174009626\n",
      "\n",
      "Best pipeline: ExtraTreesClassifier(PolynomialFeatures(input_matrix, degree=2, include_bias=False, interaction_only=False), bootstrap=True, criterion=gini, max_features=0.35000000000000003, min_samples_leaf=6, min_samples_split=16, n_estimators=100)\n",
      "0.5217391304347826\n"
     ]
    }
   ],
   "source": [
    "tpot_enabled = False\n",
    "if cell_enabled:\n",
    "    tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, random_state=42)\n",
    "    tpot.fit(X_train, y_train)\n",
    "    print(tpot.score(X_test, y_test))\n",
    "    tpot.export('tpot_npf_pipeline.py')\n",
    "    y_pred = tpot.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.65      0.92      0.76        36\n",
      "           1       0.38      0.32      0.35        28\n",
      "           2       0.35      0.25      0.29        24\n",
      "           3       0.00      0.00      0.00         4\n",
      "\n",
      "    accuracy                           0.52        92\n",
      "   macro avg       0.34      0.37      0.35        92\n",
      "weighted avg       0.46      0.52      0.48        92\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\makes\\miniconda3\\envs\\ds\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1308: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\makes\\miniconda3\\envs\\ds\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1308: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\makes\\miniconda3\\envs\\ds\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1308: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVsAAAEGCAYAAADL3zbEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiNUlEQVR4nO3deZhcZZn38e+vO52drJ2EEIIJEIMRIUBAIgMmLIKoE1C8wGGUERVBGETR14V5lZFXBkfQUQQ1ICOyCghD2MMWMSMhmzFAEgiEJEA6kA7ZyNrL/f5Rp6ETkq7q7qo6VZXf57rO1XVOnXrOXV3J3U895zn3UURgZmaFVZV2AGZmuwMnWzOzInCyNTMrAidbM7MicLI1MyuCLmkHUE5qB1THiOE1aYeRd4sX9Ek7hMJpbk47goKICn1fABtYUx8RgzrTxokTe8Xqt5qy7jdn/tZHIuKkzhwrV0627TBieA0zHxmedhh5d/LBJ6QdQsHExo1ph1AQzZs2pR1CwTwWdy3rbBv1bzXxzCN7Z92vZujLtZ09Vq6cbM2sAgVNUVq9fydbM6s4ATRTWhdsOdmaWUVqxj1bM7OCCoIGDyOYmRVWAE0eRjAzKzyP2ZqZFVgATSVW0dDJ1swqUmmN2DrZmlkFCsJjtmZmhRYBDaWVa51szawSiSaUdhDbcbI1s4oTQLN7tmZmheeerZlZgWUuanCyNTMrqAAaorTujVBa0ZiZ5UEgmqjKumQjqbukmZL+Lul5Sf+ebB8g6VFJi5Of/bO15WRrZhWpOZR1ycFW4NiIOBgYC5wk6Ujgu8DjETEKeDxZb5OTrZlVnJYx22xL1nYy3k5Wa5IlgEnAjcn2G4FTsrXlZGtmFUg0RVXWBaiVNLvVcs57WpKqJc0D3gQejYhngCERUQeQ/BycLSKfIDOzipO5U0NOfcn6iBjXZlsRTcBYSf2AeyQd2JGYnGzNrOJEiG1Rnec2Y62kacBJwBuShkZEnaShZHq9bXKyLQHbtoiLP70/DduqaGqEoz+xji98eyU3/ueePP1IXyToV9vAt/5rOQP3bEw73A6pHbKFi3/8PP0HbiVCPHzXMO69dZ+0w8qLb/zHSxxx7BrWrq7hvJPHph1OXo2bsJ5zL1tBdVXw0G0DuONXQ9IOKWfNeZhnK2kQ0JAk2h7A8cBPgCnAWcAVyc97s7XlZAtIugiYHBGp3B+6plvwn3e+TI9ezTQ2wDdPGcXhx67ntPPe5Kz/sxKA/7m+lpt/vidf/8lraYTYaU1N4vorR/Hyoj706NnIL2+fydwZA3h1Se+0Q+u0R+8ezJSb9+RbP30p7VDyqqoqOP/y1/neGftSX1fD1Q8uZsYjfVm+uHvaoWWVOUGWl1NSQ4EbJVWTOcd1R0TcL+lp4A5JXwKWA5/N1pCTbcZFwM1AKslWgh69MtU3GxtEU4OQoNce71bk3LK5CpXWBTHtsqa+G2vquwGweVMXli/pSe3grRWRbJ+b1YfBw7akHUbejT5kEyuWdmXl8sznNu3efow/cV1ZJNuWE2SdFRHzgUN2sn01cFx72irYbARJIyQtlHRdMhl4qqQeksZKmiFpvqR7WiYDS5om6SfJBOIXJR2dbK+W9FNJs5LXfDXZ/kdJJ7c63u8lfaaN/Sckx7hL0iJJtyjjQmAv4ElJTxbq95FNUxOcd/xoTj/oQA45ZgMHHJrJ+/99xZ6cedgYnri7P1/4dl1a4eXV4L02s98BG1j0bN+0Q7E2DNyzgVUrur6zXl9XQ+3QhhQjyl3LCbJsSzEV+mijgGsi4oPAWuAzwB+A70TEQcCzwA9b7d8lIo4g09Ns2f4lYF1EHA4cDnxF0kjgduB0AEldyfyVebCN/SHzF+oiYAywL3BURPwSWAFMjIiJ+f4F5Kq6Gn792AvcMmcBL8zrydJFmd7DF7+7klvmLODYT69hyg2D0govb7r3aOSSq+Yz+aej2bzRX6xK2c6+SZXYnWba1BTKuhRToZPtKxExL3k8B9gP6BcRf0623Qgc02r/u1vtOyJ5/DHgC8k8t2eAgWSS+EPAsZK6AR8HnoqIzW3sDzAzIl6LiGZgXqtj7JKkc1rm4K1a3ZTzG++o3n2bOHj828x6co/ttk88dQ3THyzvnmB1l2Yu+dl8pj24J399POu0REtZfV0Ng/ba9s567dAGVq+sSTGi3AWiIbpkXYqp0Ml2a6vHTUC/HPdv4t3xZAH/GhFjk2VkREyNiC3ANOBEMj3c29vafxfxZP1tR8TkiBgXEeMGDczvVJIWa1dX8/a6TNtbN4u5f9mD4ftv5fUl736Fm/FIX4bvv3VXTZSB4KJLF/Dqkl7cc9P70g7GcvDCvJ4MG7mNIcO30qWmmQmT1jJjann8wW85QdbZ2gj5VOzvceuANZKOjoi/AJ8H/pzlNY8A50l6IiIaJL0feD0iNpJJsF8GxgH/0tb+WY6xAdgDqO/Qu+qkt96o4cqv70Nzs2huhmM+tZYjT1jPj748gtde7kZVFQweto0Ly3QmAsCYQ9Zx3KdW8sqLvbn6jzMAuPHq/Zk9vTblyDrvOz9/kYM+vJ4+/Ru5afocbvrF3ky9s3ymSO1Kc5O45pJhXH7rEqqqYertA1j2YjmcHEsK0RR5mCCbNAbNzgJ+I6knsAT4Ypb9ryfzdX+uJAGrePc65KlkxoCnRMS2HPbflcnAQ5Lq0hi33XfMFq599MX3bP/B9UuLHUrBLPhbP04++Pi0wyiIn3zj/WmHUDCznujDrCf6pB1GhxT7BFg2BUu2EbEUOLDV+pWtnj5yJ/tPaPW4nmQ8NRlf/X6y7PiaBjJjsq237Wr/acnSst8FrR5fDVzd9jsys3IRQV6mfuWTTwebWcXJnCArzDmWjnKyNbOKVOwTYNk42ZpZxQlyLg5eNE62ZlaR3LM1MyuwAJp9gszMrNByu+1NMTnZmlnFydzK3LMRzMwKKkIeRjAzKwZf1GBmVmCZerYeszUzK7D83Kkhn5xszaziZKZ+uWdrZlZQro1gZlYku02JRTOztGRKLHoYwcys4Dxma2ZWYJmqX6U1jFBa0ZiZ5UHmct2qrEs2koZLelLSQknPS/p6sv1SSa9LmpcsJ2dryz1bM6tAeevZNgIXR8RcSXsAcyQ9mjz38x1u99UmJ1szq0j5uIIsIuqAuuTxBkkLgWEdacvDCGZWcVpmI2RbgFpJs1st5+yqTUkjgEOAZ5JNF0iaL+kGSf2zxeSebTssWDGIwy49L+0w8q77xEg7hIKp3tqcdggF0ePemWmHUPJyHEaoj4hx2XaS1Bv4E3BRRKyX9GvgMjLDw5cBVwFnt9WGk62ZVZx83oNMUg2ZRHtLRNwNEBFvtHr+OuD+bO042ZpZxQmgMQ8nyCQJ+B2wMCJ+1mr70GQ8F+BU4LlsbTnZmllFytNshKOAzwPPSpqXbPs+8DlJY8nk9aXAV7M15GRrZpUn8jOMEBHTYafTGh5sb1tOtmZWcVw83MysSFwbwcyswFw83MysCALR2Fxa12w52ZpZRfKYrZlZoYWHEczMCs5jtmZmReJka2ZWYIFo8gkyM7PC8wkyM7MCC58gMzMrjnCyNTMrtPzVs80XJ1szq0ju2ZqZFVgENDU72ZqZFZxnI5iZFVjgYQQzsyLwCTIzs6KISDuC7TnZloAfTHqSo9+/jLc29uD0a08H4PgxL3POhNmMHLSGL1z3aRauGJxylJ332WOe5R/HL0TAlBkHcMefD0o7pLz57LHP8smjFhGIJa8P4Io/HMO2xvL/7zVuwnrOvWwF1VXBQ7cN4I5fDUk7pJyV2jBCaV08nAJJbyc/R0jKejviQrhv3mj+9eZPbLftpTcH8O0/nsjcZUPTCCnvRu75Fv84fiFf/tmpnPXT0/jImOXsXbsu7bDyorbvRk6b+BxfueJU/uWy06iqaubYcUvSDqvTqqqC8y9/nX87cyRfmTCaiZPWss+oLWmHlZPMbISqrEsx7fbJthT8bdlerNvcbbttS+v7s2x1v3QCKoARQ9bw/NIhbG2ooam5inkvD+WYg15JO6y8qa4KutU0Ul3VTPeujaxe1zPtkDpt9CGbWLG0KyuXd6OxoYpp9/Zj/Inl8wcyIvtSTE62VhRLVg7g4P3q6NNzC91qGhg/ZjlD+r2ddlh5Ub+uF7c/dhB3/vg27rniFjZu7sqshXunHVanDdyzgVUrur6zXl9XQ+3QhhQjap8IZV2KqfwHlQpM0jnAOQA1vfunHE35WvZGf255fCz/dd4DbN7WhZdeH1hyJfA6qnfPrfzDwUs5/f+ewdubuvGjrzzGCUcs5tGZo9IOrVO0k1xUaieddiUofjLNpjL+tRdQREyOiHERMa5L915ph1PW7n/mAM6+6jOcf/Uk1m/qxqur+qYdUl6MO+B16ur3YN3bPWhqruKpeSM4cN830g6r0+rrahi017Z31muHNrB6ZU2KEbVP5LBkI2m4pCclLZT0vKSvJ9sHSHpU0uLkZ9aemJOtFU2/3psBGNJvAx89aCmPzd0/5Yjy4423ejNm5Jt0q2kEgsMOWMGylf3SDqvTXpjXk2EjtzFk+Fa61DQzYdJaZkwtkz+QAdGsrEsOGoGLI+IDwJHA+ZLGAN8FHo+IUcDjyXqbPIxQAn78mccYN2IF/Xpu4cFv3sRvnxzH+s3d+fbJ0+nfczO/+KeHeHHlQC64+ZNph9opl39xKn16baGxqYqr7jqKDTucFCxXC5cOZtrf9uX6799NU3MVi18dyH3TP5B2WJ3W3CSuuWQYl9+6hKpqmHr7AJa92D3tsHKWj2GEiKgD6pLHGyQtBIYBk4AJyW43AtOA77TVlpNtCbjkT8fvdPuTi0YWOZLC+trVk9IOoWD++/7D+O/7D0s7jLyb9UQfZj3RJ+0wOiTH8eVaSbNbrU+OiMk721HSCOAQ4BlgSJKIiYg6SVknwu8y2Uq6mjaGNSLiwmyNl4OI6J38XAocmG40ZpYP7aiNUB8R47LtJKk38CfgoohYr52dPcyirZ7t7DaeMzMrXQHkaTaCpBoyifaWiLg72fyGpKFJr3Yo8Ga2dnaZbCPixh0O2CsiNnYmaDOzYsnHNDVlurC/AxZGxM9aPTUFOAu4Ivl5b7a2ss5GkDRe0gJgYbJ+sKRrOxK4mVlxZJ+JkONshKOAzwPHSpqXLCeTSbInSFoMnJCstymXE2T/BZxIJpMTEX+XdEwuUZqZpSYPPduImA67rEJ+XHvaymk2QkS8usOAcFN7DmJmVlRRelW/ckm2r0r6CBCSugIXkgwpmJmVrBK7tDiXK8jOBc4nM5H3dWBssm5mVsKUw1I8WXu2EVEPnFmEWMzM8qc57QC2l8tshH0l3SdplaQ3Jd0rad9iBGdm1iEt82yzLUWUyzDCrcAdwFBgL+BO4LZCBmVm1lnlWDxcEXFTRDQmy82U3NCzmdkO8lFjMY/aqo0wIHn4pKTvAreTCe904IEixGZm1nFlNPVrDpnk2hLxV1s9F8BlhQrKzKyzVGLfv9uqjVBZ9f3MbPcRgtwuxy2anK4gk3QgMAZ4p3JwRPyhUEGZmXVaufRsW0j6IZmK5GOAB4GPA9MBJ1szK10llmxzmY1wGpmCCysj4ovAwUBl3M/EzCpXucxGaGVzRDRLapTUh0yRXF/UYGalK4/Fw/Mll2Q7W1I/4DoyMxTeBmYWMigzs84qm9kILSLia8nD30h6GOgTEfMLG5aZWSeVS7KVdGhbz0XE3MKEZGbWeeXUs72qjecCODbPsZS8LluaGbBgc9ph5N2qQ3umHULBDJq7Je0QLC3lMmYbEROLGYiZWd6kMNsgm5wuajAzKztOtmZmhacSKx7uZGtmlanEera53KlBkv5Z0g+S9X0kHVH40MzMOkaR21JMuVyuey0wHvhcsr4BuKZgEZmZ5UMZ3hbnwxFxPrAFICLWAF0LGpWZWWflqTaCpBuS+y8+12rbpZJelzQvWU7O1k4uybZBUnVLaJIGUXL3rTQz214ehxF+D5y0k+0/j4ixyfJgtkZySba/BO4BBkv6MZnyipfnHKaZWbFFZjZCtiWnpiKeAt7qbEi51Ea4RdIcMmUWBZwSEQs7e2Azs4LKredaK2l2q/XJETE5xyNcIOkLwGzg4mSIdZdyKR6+D7AJuK/1tohYnmNAZmbFl1uyrY+IcR1o/ddk7sPYcj/Gq4Cz23pBLvNsH+DdGz92B0YCLwAf7ECAZmZFUcipXRHxxjvHka4D7s/2mlyGET7Uej2pBvbVXexuZlbxJA2NiLpk9VTgubb2hw5cQRYRcyUd3t7XmZkVVZ56tpJuI3MfxlpJrwE/BCZIGpscZSk5dEBzGbP9ZqvVKuBQYFW7IzYzK5bIX22EiPjcTjb/rr3t5NKz3aPV40YyY7h/au+BzMyKqsRqI7SZbJOLGXpHxLeLFI+ZWaeJMrpTg6QuEdHY1u1xzMxKVrkkWzJ30D0UmCdpCnAnsLHlyYi4u8CxmZl1TApVvbLJZcx2ALCazD3HWubbBuBka2alq8QquLSVbAcnMxGe490k26LE/maYmW2vnHq21UBvtk+yLUrsbZiZ7aDEslRbybYuIn5UtEjsHb16buOb5/6VEcPXQIgrf/0RFi4enHZYHXLpyU9yzH5LeWtTD0773RkAfGPiXzlm/2U0NFXx2tq+/PCBiWzY2i3lSDunkj6z1sZNWM+5l62guip46LYB3PGrIWmHlJsyu7tuad10PU8kvR0RvSVNAL4VEZ9MOaT3+NoXZzJ73l5c9rMJdKluolu3prRD6rApz47m9jkH8v8++fg722a8MpxfTjuSpqji6xOe5uzxc/nFtPEpRtl5lfSZtaiqCs6//HW+d8a+1NfVcPWDi5nxSF+WL+6edmg5KbVhhLbq2R5XtCjsHT17bONDH3iDh54YBUBjUzUbN5XvjTHmvroX67ds32t9eulwmiLzT2/+iiEM2WPjzl5aNirtM2sx+pBNrFjalZXLu9HYUMW0e/sx/sR1aYeVuzzdqSFfdtmzjYhOF8stA30k3QOMBp4CvhYRqZ7DHDr4bdat78a3v/a/7Pu+NSxeMpBrf384W7bWpBlWwZxy0CIeWbh/2mF0SqV+ZgP3bGDVinf/aNTX1XDAoZtSjKh9Su1W5rncqaGSHQFcDHwI2A/49I47SDpH0mxJs7c1FL4HVl3dzKiRb3Hf1NGc951PsWVrF04/JWtBobL05fFzaGqu4sHnR6UdSqdU6memnQwkRol9Nd+lXHq1JXh33Uo2MyKWREQTcBvwDzvuEBGTI2JcRIzrWtOr4AGtWt2LVat7suilQQA8NeN9jBq5uuDHLbZPHbiIo/dfxventNwApHxV6mdWX1fDoL22vbNeO7SB1SvLo7euHJdi2t2T7Y5/21L/u71mXQ9Wre7F3kMzY2OHfKiOZa/1SzeoPPvIyOX8y5HzuOiuj7OlsTz+87alUj+zF+b1ZNjIbQwZvpUuNc1MmLSWGVP7ph1W7kqsZ9vuerYV5ghJI4FlwOlArvceKqhrbvgw37vwL3Tp0kzdm7258tqj0g6pw/7jHx9l3D4r6NdjC4987Q/8evrhnD1+Ll2rm/jNGZk7Lc1fMYQfP/LRlCPtnEr6zFo0N4lrLhnG5bcuoaoapt4+gGUvlsdMBCi92Qi7e7J9GriCzJjtU2TuIpy6l5cN4PzvldyMtA753pQT3rPtf+Z/IIVICquSPrPWZj3Rh1lP9Ek7jI5xsk1XRPROfk4DpqUajJkVRh6Lh+fLbpdszWw34Z6tmVnheczWzKwYnGzNzArPPVszs0ILyqp4uJlZWSqrGz6amZW1Eku2u/vlumZWoRSRdcmpHekGSW9Keq7VtgGSHpW0OPnZP1s7TrZmVnnyW/Xr98BJO2z7LvB4RIwCHk/W2+Rka2YVSZF9yUVEPAXsWN97EnBj8vhG4JRs7XjM1swqUo6X69ZKmt1qfXJE5FKQakhE1AFERJ2krDecc7I1s8qUW8+1PiLGFTgSwMMIZlaJchhC6OTUsDckDQVIfr6Z7QVOtmZWmQpbPHwKcFby+Czg3mwvcLI1s4rTclFDPnq2km4jU/t6tKTXJH2JTB3sEyQtBk5I1tvkMVszq0hqzs9VDRHxuV08dVx72nGyNbPKk8I9xrJxsjWziuQ7NZiZFYN7tmZmheeqX2ZmhRZAjoVmisXJth20aStd5r6Ydhh5N3Ru2hEUjnr1SjuEgmhKO4Ay4DFbM7MCc/FwM7NiiPAwgplZMbhna2ZWDE62ZmaF556tmVmhBdBUWtnWydbMKpJ7tmZmxeDZCGZmheeerZlZobnEoplZ4QmQT5CZmRWePGZrZlZgHkYwMysG10YwMysKz0YwMysG92zNzAosPBvBzKw4SivXOtmaWWXK19QvSUuBDWTuRtQYEeM60o6TrZlVpvyO2U6MiPrONOBka2aVJ4ASu+FjVdoBmJnlmwgU2RegVtLsVss5O2kugKmS5uzi+Zy4Z1tivvEfL3HEsWtYu7qG804em3Y4eVWp7612yBYu/vHz9B+4lQjx8F3DuPfWfdIOKy/GTVjPuZetoLoqeOi2AdzxqyFph5S75py6tvU5jMEeFRErJA0GHpW0KCKeam84u13PVtLbacfQlkfvHsy/nf2BtMMoiEp9b01N4vorR3HuqR/hm/98OJ884zWG71vS/8xyUlUVnH/56/zbmSP5yoTRTJy0ln1GbUk7rNy0DCNkW3JpKmJF8vNN4B7giI6EtNsl21L33Kw+bFhbmV84KvW9ranvxsuL+gCweVMXli/pSe3grSlH1XmjD9nEiqVdWbm8G40NVUy7tx/jT1yXdlg5y3EYoe02pF6S9mh5DHwMeK4j8ey2yVZSb0mPS5or6VlJk9KOycrf4L02s98BG1j0bN+0Q+m0gXs2sGpF13fW6+tqqB3akGJE7RSRfcluCDBd0t+BmcADEfFwR8KpvG5G7rYAp0bEekm1wAxJUyJK7Bo/KxvdezRyyVXzmfzT0WzeWP7/taT3biuf/x35KUQTEUuAgzsfz+6dbAVcLukYMqM3w8j8FVu53U6Zs4/nAHRXr2LHaGWiukszl/xsPtMe3JO/Pj447XDyor6uhkF7bXtnvXZoA6tX1qQYUTuU4N11d9thBOBMYBBwWESMBd4Auu+4U0RMjohxETGuq97ztBkQXHTpAl5d0ot7bnpf2sHkzQvzejJs5DaGDN9Kl5pmJkxay4yp5TM8ko8x23zanXu2fYE3I6JB0kSgJP6XfOfnL3LQh9fTp38jN02fw02/2Jupd5bRdJs2VOp7G3PIOo771EpeebE3V/9xBgA3Xr0/s6fXphxZ5zQ3iWsuGcblty6hqhqm3j6AZS+WUYejxMY8dudkewtwn6TZwDxgUbrhZPzkG+9PO4SCqdT3tuBv/Tj54OPTDqMgZj3Rh1lP9Ek7jPYLoNnJNlUR0Tv5WQ+MTzkcMysI36nBzKw4nGzNzAosgKbSqkTjZGtmFSggnGzNzArPwwhmZgXm2QhmZkXinq2ZWRE42ZqZFVgENDWlHcV2nGzNrDK5Z2tmVgROtmZmhRaejWBmVnAB4YsazMyKwJfrmpkVWESutzIvGidbM6tMPkFmZlZ44Z6tmVmhuXi4mVnhuRCNmVnhBRAldrnu7nwrczOrVJEUD8+25EDSSZJekPSSpO92NCT3bM2sIkUehhEkVQPXACcArwGzJE2JiAXtbcs9WzOrTPnp2R4BvBQRSyJiG3A7MKkj4ShK7IxdKZO0ClhWxEPWAvVFPF6x+H2Vn2K+t/dFxKDONCDpYTIxZ9Md2NJqfXJETG7VzmnASRHx5WT988CHI+KC9sbkYYR26Ow/gPaSNDsixhXzmMXg91V+yu29RcRJeWpKO2u+Iw15GMHMbNdeA4a3Wt8bWNGRhpxszcx2bRYwStJISV2BM4ApHWnIwwilbXL2XcqS31f5qeT3tksR0SjpAuARoBq4ISKe70hbPkFmZlYEHkYwMysCJ1szsyJwst2NSLpIUs8Uj/928nOEpOfSiiOfWr2nCZLuTzuefGl5X5Y/Tra7l4uA1JKt2e7MybaTkl7aQknXSXpe0lRJPSSNlTRD0nxJ90jqn+w/TdJPJM2U9KKko5Pt1ZJ+KmlW8pqvJtv/KOnkVsf7vaTPtLH/hOQYd0laJOkWZVwI7AU8KenJ4v+mdgt9ks96gaTfSCr7/1+Sekt6XNJcSc9K6tClquZkmy+jgGsi4oPAWuAzwB+A70TEQcCzwA9b7d8lIo4g09Ns2f4lYF1EHA4cDnxF0kgy12KfDpDM8zsOeLCN/QEOSdoeA+wLHBURvyQzGXtiREzM9y/AgMx19BcDHwL2Az6dbjh5sQU4NSIOBSYCV0na2VVVloWTbX68EhHzksdzyPxH6xcRf0623Qgc02r/u1vtOyJ5/DHgC5LmAc8AA8kk8YeAYyV1Az4OPBURm9vYH2BmRLwWmXs5z2t1DCusmUnBkibgNuAf0g4oDwRcLmk+8BgwDBiSbkjlyRc15MfWVo+bgH457t/Eu5+BgH+NiEd23FnSNOBEMj3c29raX9KEncTjz7k4dpy0XgmT2M8EBgGHRUSDpKVkirdYO7lnWxjrgDUt47HA54E/t7E/ZK5QOU9SDYCk90vqlTx3O/BF4Ohkv2z778oGYI92vRNrjyOSyzqryPxhnJ52QHnQF3gzSbQTgfelHVC5co+ncM4CfpNMtVpCJlm25XoyX/fnJmNiq4BTkuemkhkDnpLU1My2/65MBh6SVOdx24J4GriCzJjtU8A96YaTF7cA90maTWZIalG64ZQvX65rZlYEHkYwMysCJ1szsyJwsjUzKwInWzOzInCyNTMrAidbyytJTZLmSXpO0p2dqTKW1IE4LXl8vaQxbew7QdJHOnCMpZLecxfWXW3fYZ92VcaSdKmkb7U3RqsMTraWb5sjYmxEHAhsA85t/aSk6o40GhFfjogFbewyAWh3sjUrFidbK6S/APsnvc4nJd0KPNtGxTJJ+lVSNesBYHBLQ0kls3HJ45OSKlR/TypSjSCT1L+R9KqPljRI0p+SY8ySdFTy2oFJZba/SfotO79V9XYk/Y+kOcpUdTtnh+euSmJ5XNKgZNt+kh5OXvMXSQfk5bdpZc1XkFlBSOpCpnDOw8mmI4ADI+KVJGGti4jDkwI7/ytpKplqZaPJXIE1BFgA3LBDu4OA64BjkrYGRMRbkn4DvB0RVyb73Qr8PCKmS9qHzOXNHyBTZW16RPxI0ieA7ZLnLpydHKMHMEvSnyJiNdALmBsRF0v6QdL2BWSu1Ds3IhZL+jBwLXBsB36NVkGcbC3feiSVyCDTs/0dma/3MyPilWT7x4CDWsZjyVx/P4pMZbTbkqpZKyQ9sZP2jyRT+ewVgIh4axdxHA+MaVUNsI+kPZJjfDp57QOS1uTwni6UdGryeHgS62qgGfhjsv1m4G5JvZP3e2erY3fL4RhW4ZxsLd82R8TY1huSpLOx9SZ2XrHsZLJXylIO+0BmiGx8Uo5yx1hyvkY9qaJ2fNLWpqQC266qXkVy3LU7/g7MPGZradhVxbKngDOSMd2hZIpV7+hp4KMthdIlDUi271jRbCqZr/Qk+41NHj5Fpmwgkj4O9M8Sa19gTZJoDyDTs25RBbT0zv+JzPDEeuAVSZ9NjiFJB2c5hu0GnGwtDdeTGY+dq8yNH39L5lvWPcBiMne2+DU7KUsZEavIjLPeLenvvPs1/j7g1JYTZMCFwLjkBNwC3p0V8e/AMZLmkhnOWJ4l1oeBLsoUz74MmNHquY3AByXNITMm+6Nk+5nAl5L4ngd8Kxlz1S8zs2Jwz9bMrAicbM3MisDJ1sysCJxszcyKwMnWzKwInGzNzIrAydbMrAj+P1xlPiw04+waAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if tpot_enabled:\n",
    "    print_classification_metrics(y_test, y_pred)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2c1fc70ff3ed5b375ef3c0c998544f7de210ad65b9844929bce5458d51796adc"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit ('ds': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}