KNN_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sklearn in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (from sklearn) (0.24.2)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (1.0.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (1.19.2)\n",
      "Requirement already satisfied: scipy>=0.19.1 in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (1.6.3)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (2.1.0)\n"
     ]
    }
   ],
   "source": [
    "# Update sklearn to prevent version mismatches\n",
    "!pip install sklearn --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: joblib in c:\\users\\nikoe\\anaconda3\\lib\\site-packages (1.0.1)\n"
     ]
    }
   ],
   "source": [
    "# install joblib. This will be used to save your model. \n",
    "# Restart your kernel after installing \n",
    "!pip install joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the CSV and Perform Basic Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_disposition</th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_period_err1</th>\n",
       "      <th>koi_period_err2</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_time0bk_err1</th>\n",
       "      <th>...</th>\n",
       "      <th>koi_steff_err2</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_slogg_err1</th>\n",
       "      <th>koi_slogg_err2</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>koi_srad_err1</th>\n",
       "      <th>koi_srad_err2</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_kepmag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>54.418383</td>\n",
       "      <td>2.479000e-04</td>\n",
       "      <td>-2.479000e-04</td>\n",
       "      <td>162.513840</td>\n",
       "      <td>0.003520</td>\n",
       "      <td>...</td>\n",
       "      <td>-81</td>\n",
       "      <td>4.467</td>\n",
       "      <td>0.064</td>\n",
       "      <td>-0.096</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.105</td>\n",
       "      <td>-0.061</td>\n",
       "      <td>291.93423</td>\n",
       "      <td>48.141651</td>\n",
       "      <td>15.347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.899140</td>\n",
       "      <td>1.490000e-05</td>\n",
       "      <td>-1.490000e-05</td>\n",
       "      <td>175.850252</td>\n",
       "      <td>0.000581</td>\n",
       "      <td>...</td>\n",
       "      <td>-176</td>\n",
       "      <td>4.544</td>\n",
       "      <td>0.044</td>\n",
       "      <td>-0.176</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.233</td>\n",
       "      <td>-0.078</td>\n",
       "      <td>297.00482</td>\n",
       "      <td>48.134129</td>\n",
       "      <td>15.436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.736952</td>\n",
       "      <td>2.630000e-07</td>\n",
       "      <td>-2.630000e-07</td>\n",
       "      <td>170.307565</td>\n",
       "      <td>0.000115</td>\n",
       "      <td>...</td>\n",
       "      <td>-174</td>\n",
       "      <td>4.564</td>\n",
       "      <td>0.053</td>\n",
       "      <td>-0.168</td>\n",
       "      <td>0.791</td>\n",
       "      <td>0.201</td>\n",
       "      <td>-0.067</td>\n",
       "      <td>285.53461</td>\n",
       "      <td>48.285210</td>\n",
       "      <td>15.597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.525592</td>\n",
       "      <td>3.760000e-06</td>\n",
       "      <td>-3.760000e-06</td>\n",
       "      <td>171.595550</td>\n",
       "      <td>0.001130</td>\n",
       "      <td>...</td>\n",
       "      <td>-211</td>\n",
       "      <td>4.438</td>\n",
       "      <td>0.070</td>\n",
       "      <td>-0.210</td>\n",
       "      <td>1.046</td>\n",
       "      <td>0.334</td>\n",
       "      <td>-0.133</td>\n",
       "      <td>288.75488</td>\n",
       "      <td>48.226200</td>\n",
       "      <td>15.509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.134435</td>\n",
       "      <td>1.050000e-05</td>\n",
       "      <td>-1.050000e-05</td>\n",
       "      <td>172.979370</td>\n",
       "      <td>0.001900</td>\n",
       "      <td>...</td>\n",
       "      <td>-232</td>\n",
       "      <td>4.486</td>\n",
       "      <td>0.054</td>\n",
       "      <td>-0.229</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.315</td>\n",
       "      <td>-0.105</td>\n",
       "      <td>296.28613</td>\n",
       "      <td>48.224670</td>\n",
       "      <td>15.714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  koi_disposition  koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  \\\n",
       "0       CONFIRMED              0              0              0              0   \n",
       "1  FALSE POSITIVE              0              1              0              0   \n",
       "2  FALSE POSITIVE              0              1              0              0   \n",
       "3       CONFIRMED              0              0              0              0   \n",
       "4       CONFIRMED              0              0              0              0   \n",
       "\n",
       "   koi_period  koi_period_err1  koi_period_err2  koi_time0bk  \\\n",
       "0   54.418383     2.479000e-04    -2.479000e-04   162.513840   \n",
       "1   19.899140     1.490000e-05    -1.490000e-05   175.850252   \n",
       "2    1.736952     2.630000e-07    -2.630000e-07   170.307565   \n",
       "3    2.525592     3.760000e-06    -3.760000e-06   171.595550   \n",
       "4    4.134435     1.050000e-05    -1.050000e-05   172.979370   \n",
       "\n",
       "   koi_time0bk_err1  ...  koi_steff_err2  koi_slogg  koi_slogg_err1  \\\n",
       "0          0.003520  ...             -81      4.467           0.064   \n",
       "1          0.000581  ...            -176      4.544           0.044   \n",
       "2          0.000115  ...            -174      4.564           0.053   \n",
       "3          0.001130  ...            -211      4.438           0.070   \n",
       "4          0.001900  ...            -232      4.486           0.054   \n",
       "\n",
       "   koi_slogg_err2  koi_srad  koi_srad_err1  koi_srad_err2         ra  \\\n",
       "0          -0.096     0.927          0.105         -0.061  291.93423   \n",
       "1          -0.176     0.868          0.233         -0.078  297.00482   \n",
       "2          -0.168     0.791          0.201         -0.067  285.53461   \n",
       "3          -0.210     1.046          0.334         -0.133  288.75488   \n",
       "4          -0.229     0.972          0.315         -0.105  296.28613   \n",
       "\n",
       "         dec  koi_kepmag  \n",
       "0  48.141651      15.347  \n",
       "1  48.134129      15.436  \n",
       "2  48.285210      15.597  \n",
       "3  48.226200      15.509  \n",
       "4  48.224670      15.714  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"exoplanet_data.csv\")\n",
    "# Drop the null columns where all values are null\n",
    "df = df.dropna(axis='columns', how='all')\n",
    "# Drop the null rows\n",
    "df = df.dropna()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select your features (columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set features. This will also be used as your x values.\n",
    "selected_features = df.drop(\"koi_disposition\", axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a Train Test Split\n",
    "\n",
    "Use `koi_disposition` for the y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0            CONFIRMED\n",
       "1       FALSE POSITIVE\n",
       "2       FALSE POSITIVE\n",
       "3            CONFIRMED\n",
       "4            CONFIRMED\n",
       "             ...      \n",
       "6986    FALSE POSITIVE\n",
       "6987    FALSE POSITIVE\n",
       "6988         CANDIDATE\n",
       "6989    FALSE POSITIVE\n",
       "6990    FALSE POSITIVE\n",
       "Name: koi_disposition, Length: 6991, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = df[\"koi_disposition\"]\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(selected_features, y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_period_err1</th>\n",
       "      <th>koi_period_err2</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_time0bk_err1</th>\n",
       "      <th>koi_time0bk_err2</th>\n",
       "      <th>...</th>\n",
       "      <th>koi_steff_err2</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_slogg_err1</th>\n",
       "      <th>koi_slogg_err2</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>koi_srad_err1</th>\n",
       "      <th>koi_srad_err2</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_kepmag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6122</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.768901</td>\n",
       "      <td>7.380000e-05</td>\n",
       "      <td>-7.380000e-05</td>\n",
       "      <td>133.077240</td>\n",
       "      <td>0.008440</td>\n",
       "      <td>-0.008440</td>\n",
       "      <td>...</td>\n",
       "      <td>-171</td>\n",
       "      <td>4.327</td>\n",
       "      <td>0.153</td>\n",
       "      <td>-0.187</td>\n",
       "      <td>1.125</td>\n",
       "      <td>0.310</td>\n",
       "      <td>-0.207</td>\n",
       "      <td>294.40472</td>\n",
       "      <td>39.351681</td>\n",
       "      <td>14.725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6370</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.733726</td>\n",
       "      <td>6.060000e-06</td>\n",
       "      <td>-6.060000e-06</td>\n",
       "      <td>132.020050</td>\n",
       "      <td>0.007950</td>\n",
       "      <td>-0.007950</td>\n",
       "      <td>...</td>\n",
       "      <td>-175</td>\n",
       "      <td>4.578</td>\n",
       "      <td>0.033</td>\n",
       "      <td>-0.187</td>\n",
       "      <td>0.797</td>\n",
       "      <td>0.211</td>\n",
       "      <td>-0.056</td>\n",
       "      <td>284.50391</td>\n",
       "      <td>42.463860</td>\n",
       "      <td>15.770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2879</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.652707</td>\n",
       "      <td>6.540000e-05</td>\n",
       "      <td>-6.540000e-05</td>\n",
       "      <td>134.460380</td>\n",
       "      <td>0.006190</td>\n",
       "      <td>-0.006190</td>\n",
       "      <td>...</td>\n",
       "      <td>-189</td>\n",
       "      <td>4.481</td>\n",
       "      <td>0.050</td>\n",
       "      <td>-0.200</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.290</td>\n",
       "      <td>-0.097</td>\n",
       "      <td>295.50211</td>\n",
       "      <td>38.983540</td>\n",
       "      <td>13.099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.953547</td>\n",
       "      <td>1.910000e-05</td>\n",
       "      <td>-1.910000e-05</td>\n",
       "      <td>174.662240</td>\n",
       "      <td>0.001820</td>\n",
       "      <td>-0.001820</td>\n",
       "      <td>...</td>\n",
       "      <td>-85</td>\n",
       "      <td>4.536</td>\n",
       "      <td>0.056</td>\n",
       "      <td>-0.016</td>\n",
       "      <td>0.779</td>\n",
       "      <td>0.023</td>\n",
       "      <td>-0.049</td>\n",
       "      <td>291.15878</td>\n",
       "      <td>40.750271</td>\n",
       "      <td>15.660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.959319</td>\n",
       "      <td>5.150000e-07</td>\n",
       "      <td>-5.150000e-07</td>\n",
       "      <td>172.258529</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>-0.000083</td>\n",
       "      <td>...</td>\n",
       "      <td>-77</td>\n",
       "      <td>4.359</td>\n",
       "      <td>0.110</td>\n",
       "      <td>-0.110</td>\n",
       "      <td>1.082</td>\n",
       "      <td>0.173</td>\n",
       "      <td>-0.130</td>\n",
       "      <td>292.16705</td>\n",
       "      <td>48.727589</td>\n",
       "      <td>15.263</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  koi_period  \\\n",
       "6122              0              0              0              0    6.768901   \n",
       "6370              0              1              0              1    0.733726   \n",
       "2879              1              0              0              0    7.652707   \n",
       "107               0              0              0              0    7.953547   \n",
       "29                0              0              0              0    4.959319   \n",
       "\n",
       "      koi_period_err1  koi_period_err2  koi_time0bk  koi_time0bk_err1  \\\n",
       "6122     7.380000e-05    -7.380000e-05   133.077240          0.008440   \n",
       "6370     6.060000e-06    -6.060000e-06   132.020050          0.007950   \n",
       "2879     6.540000e-05    -6.540000e-05   134.460380          0.006190   \n",
       "107      1.910000e-05    -1.910000e-05   174.662240          0.001820   \n",
       "29       5.150000e-07    -5.150000e-07   172.258529          0.000083   \n",
       "\n",
       "      koi_time0bk_err2  ...  koi_steff_err2  koi_slogg  koi_slogg_err1  \\\n",
       "6122         -0.008440  ...            -171      4.327           0.153   \n",
       "6370         -0.007950  ...            -175      4.578           0.033   \n",
       "2879         -0.006190  ...            -189      4.481           0.050   \n",
       "107          -0.001820  ...             -85      4.536           0.056   \n",
       "29           -0.000083  ...             -77      4.359           0.110   \n",
       "\n",
       "      koi_slogg_err2  koi_srad  koi_srad_err1  koi_srad_err2         ra  \\\n",
       "6122          -0.187     1.125          0.310         -0.207  294.40472   \n",
       "6370          -0.187     0.797          0.211         -0.056  284.50391   \n",
       "2879          -0.200     0.963          0.290         -0.097  295.50211   \n",
       "107           -0.016     0.779          0.023         -0.049  291.15878   \n",
       "29            -0.110     1.082          0.173         -0.130  292.16705   \n",
       "\n",
       "            dec  koi_kepmag  \n",
       "6122  39.351681      14.725  \n",
       "6370  42.463860      15.770  \n",
       "2879  38.983540      13.099  \n",
       "107   40.750271      15.660  \n",
       "29    48.727589      15.263  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-processing\n",
    "\n",
    "Scale the data using the MinMaxScaler and perform some feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale your data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "    \n",
    "X_scaler = StandardScaler().fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    " # Transform the training and testing data using the X_scaler and y_scaler models\n",
    "\n",
    "X_train_scaled = X_scaler.transform(X_train)\n",
    "X_test_scaled = X_scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dependencies for KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k: 1| TrainScore: 1.000 --- TestScore: 0.804\n",
      "k: 3| TrainScore: 0.910 --- TestScore: 0.831\n",
      "k: 5| TrainScore: 0.889 --- TestScore: 0.832\n",
      "k: 7| TrainScore: 0.880 --- TestScore: 0.830\n",
      "k: 9| TrainScore: 0.874 --- TestScore: 0.830\n",
      "k: 11| TrainScore: 0.869 --- TestScore: 0.831\n",
      "k: 13| TrainScore: 0.865 --- TestScore: 0.826\n",
      "k: 15| TrainScore: 0.861 --- TestScore: 0.824\n",
      "k: 17| TrainScore: 0.860 --- TestScore: 0.824\n",
      "k: 19| TrainScore: 0.856 --- TestScore: 0.830\n"
     ]
    }
   ],
   "source": [
    "# Loop through different k values to see which has the highest accuracy\n",
    "# Note: We only use odd numbers because we don't want any ties\n",
    "train_scores = []\n",
    "test_scores = []\n",
    "for k in range(1, 20, 2):\n",
    "    knn = KNeighborsClassifier(n_neighbors=k)\n",
    "    knn.fit(X_train_scaled, y_train)\n",
    "    train_score = knn.score(X_train_scaled, y_train)\n",
    "    test_score = knn.score(X_test_scaled, y_test)\n",
    "    train_scores.append(train_score)\n",
    "    test_scores.append(test_score)\n",
    "    print(f\"k: {k}| TrainScore: {train_score:.3f} --- TestScore: {test_score:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k=5 Test Acc: 0.832\n"
     ]
    }
   ],
   "source": [
    " # k: 5 seems to be the best choice for this dataset\n",
    "knn = KNeighborsClassifier(n_neighbors=5)\n",
    "knn.fit(X_train_scaled, y_train)\n",
    "print('k=5 Test Acc: %.3f' % knn.score(X_test_scaled, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning\n",
    "\n",
    "Use `GridSearchCV` to tune the model's parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the GridSearchCV model\n",
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11],\n",
    "              'weights': ['uniform', 'distance'],\n",
    "              'algorithm': ['auto', 'kd_tree', 'ball_tree']}\n",
    "grid = GridSearchCV(knn, param_grid, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 36 candidates, totalling 180 fits\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=1, weights=uniform;, score=0.813 total time=   0.2s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=1, weights=uniform;, score=0.803 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=1, weights=uniform;, score=0.798 total time=   0.1s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=1, weights=uniform;, score=0.806 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=1, weights=uniform;, score=0.819 total time=   0.1s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=1, weights=distance;, score=0.813 total time=   0.0s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=1, weights=distance;, score=0.803 total time=   0.0s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=1, weights=distance;, score=0.798 total time=   0.0s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=1, weights=distance;, score=0.806 total time=   0.0s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=1, weights=distance;, score=0.819 total time=   0.0s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=3, weights=uniform;, score=0.847 total time=   0.2s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=3, weights=uniform;, score=0.822 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=3, weights=uniform;, score=0.824 total time=   0.2s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=3, weights=uniform;, score=0.818 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=3, weights=uniform;, score=0.838 total time=   0.2s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=3, weights=distance;, score=0.848 total time=   0.1s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=3, weights=distance;, score=0.822 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=3, weights=distance;, score=0.825 total time=   0.1s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=3, weights=distance;, score=0.819 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=3, weights=distance;, score=0.838 total time=   0.1s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=5, weights=uniform;, score=0.836 total time=   0.2s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=5, weights=uniform;, score=0.827 total time=   0.2s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=5, weights=uniform;, score=0.828 total time=   0.2s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=5, weights=uniform;, score=0.827 total time=   0.2s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=5, weights=uniform;, score=0.842 total time=   0.2s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=5, weights=distance;, score=0.839 total time=   0.1s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=5, weights=distance;, score=0.827 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=5, weights=distance;, score=0.831 total time=   0.1s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=5, weights=distance;, score=0.827 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=5, weights=distance;, score=0.841 total time=   0.1s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=7, weights=uniform;, score=0.841 total time=   0.2s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=7, weights=uniform;, score=0.829 total time=   0.2s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=7, weights=uniform;, score=0.832 total time=   0.2s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=7, weights=uniform;, score=0.824 total time=   0.2s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=7, weights=uniform;, score=0.837 total time=   0.2s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=7, weights=distance;, score=0.840 total time=   0.1s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=7, weights=distance;, score=0.826 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=7, weights=distance;, score=0.831 total time=   0.1s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=7, weights=distance;, score=0.825 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=7, weights=distance;, score=0.839 total time=   0.1s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=9, weights=uniform;, score=0.846 total time=   0.2s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=9, weights=uniform;, score=0.827 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=9, weights=uniform;, score=0.837 total time=   0.2s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=9, weights=uniform;, score=0.824 total time=   0.2s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=9, weights=uniform;, score=0.841 total time=   0.2s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=9, weights=distance;, score=0.848 total time=   0.1s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=9, weights=distance;, score=0.828 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=9, weights=distance;, score=0.835 total time=   0.1s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=9, weights=distance;, score=0.825 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=9, weights=distance;, score=0.845 total time=   0.1s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=11, weights=uniform;, score=0.842 total time=   0.2s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=11, weights=uniform;, score=0.829 total time=   0.2s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=11, weights=uniform;, score=0.842 total time=   0.2s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=11, weights=uniform;, score=0.827 total time=   0.2s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=11, weights=uniform;, score=0.837 total time=   0.2s\n",
      "[CV 1/5] END algorithm=auto, n_neighbors=11, weights=distance;, score=0.844 total time=   0.1s\n",
      "[CV 2/5] END algorithm=auto, n_neighbors=11, weights=distance;, score=0.832 total time=   0.1s\n",
      "[CV 3/5] END algorithm=auto, n_neighbors=11, weights=distance;, score=0.839 total time=   0.1s\n",
      "[CV 4/5] END algorithm=auto, n_neighbors=11, weights=distance;, score=0.832 total time=   0.1s\n",
      "[CV 5/5] END algorithm=auto, n_neighbors=11, weights=distance;, score=0.840 total time=   0.1s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=1, weights=uniform;, score=0.813 total time=   0.5s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=1, weights=uniform;, score=0.803 total time=   0.4s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=1, weights=uniform;, score=0.798 total time=   0.4s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=1, weights=uniform;, score=0.806 total time=   0.5s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=1, weights=uniform;, score=0.819 total time=   0.5s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=1, weights=distance;, score=0.813 total time=   0.4s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=1, weights=distance;, score=0.803 total time=   0.4s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=1, weights=distance;, score=0.798 total time=   0.4s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=1, weights=distance;, score=0.806 total time=   0.4s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=1, weights=distance;, score=0.819 total time=   0.4s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=3, weights=uniform;, score=0.847 total time=   0.5s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=3, weights=uniform;, score=0.822 total time=   0.5s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=3, weights=uniform;, score=0.824 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=3, weights=uniform;, score=0.818 total time=   0.6s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=3, weights=uniform;, score=0.838 total time=   0.7s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=3, weights=distance;, score=0.848 total time=   0.7s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=3, weights=distance;, score=0.822 total time=   0.7s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=3, weights=distance;, score=0.825 total time=   0.7s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=3, weights=distance;, score=0.819 total time=   0.7s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=3, weights=distance;, score=0.838 total time=   0.7s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=5, weights=uniform;, score=0.836 total time=   0.7s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=5, weights=uniform;, score=0.827 total time=   0.9s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=5, weights=uniform;, score=0.828 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=5, weights=uniform;, score=0.827 total time=   0.8s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=5, weights=uniform;, score=0.842 total time=   0.5s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=5, weights=distance;, score=0.839 total time=   0.7s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=5, weights=distance;, score=0.827 total time=   0.5s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=5, weights=distance;, score=0.831 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=5, weights=distance;, score=0.827 total time=   0.5s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=5, weights=distance;, score=0.841 total time=   0.5s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=7, weights=uniform;, score=0.841 total time=   0.5s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=7, weights=uniform;, score=0.829 total time=   0.6s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=7, weights=uniform;, score=0.832 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=7, weights=uniform;, score=0.824 total time=   0.5s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=7, weights=uniform;, score=0.837 total time=   0.5s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=7, weights=distance;, score=0.840 total time=   0.5s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=7, weights=distance;, score=0.826 total time=   0.5s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=7, weights=distance;, score=0.831 total time=   0.4s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=7, weights=distance;, score=0.825 total time=   0.5s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=7, weights=distance;, score=0.839 total time=   0.5s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=9, weights=uniform;, score=0.846 total time=   0.6s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=9, weights=uniform;, score=0.827 total time=   0.7s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=9, weights=uniform;, score=0.837 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=9, weights=uniform;, score=0.824 total time=   0.6s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=9, weights=uniform;, score=0.841 total time=   0.6s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=9, weights=distance;, score=0.848 total time=   0.6s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=9, weights=distance;, score=0.828 total time=   0.5s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=9, weights=distance;, score=0.835 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=9, weights=distance;, score=0.825 total time=   0.6s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=9, weights=distance;, score=0.845 total time=   0.5s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=11, weights=uniform;, score=0.842 total time=   0.6s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=11, weights=uniform;, score=0.829 total time=   0.6s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=11, weights=uniform;, score=0.842 total time=   0.6s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=11, weights=uniform;, score=0.827 total time=   0.5s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=11, weights=uniform;, score=0.837 total time=   0.6s\n",
      "[CV 1/5] END algorithm=kd_tree, n_neighbors=11, weights=distance;, score=0.844 total time=   0.5s\n",
      "[CV 2/5] END algorithm=kd_tree, n_neighbors=11, weights=distance;, score=0.832 total time=   0.5s\n",
      "[CV 3/5] END algorithm=kd_tree, n_neighbors=11, weights=distance;, score=0.839 total time=   0.5s\n",
      "[CV 4/5] END algorithm=kd_tree, n_neighbors=11, weights=distance;, score=0.832 total time=   0.5s\n",
      "[CV 5/5] END algorithm=kd_tree, n_neighbors=11, weights=distance;, score=0.840 total time=   0.5s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=1, weights=uniform;, score=0.813 total time=   0.4s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=1, weights=uniform;, score=0.803 total time=   0.4s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=1, weights=uniform;, score=0.798 total time=   0.4s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=1, weights=uniform;, score=0.806 total time=   0.4s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=1, weights=uniform;, score=0.819 total time=   0.4s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=1, weights=distance;, score=0.813 total time=   0.3s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=1, weights=distance;, score=0.803 total time=   0.3s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=1, weights=distance;, score=0.798 total time=   0.3s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=1, weights=distance;, score=0.806 total time=   0.3s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=1, weights=distance;, score=0.819 total time=   0.3s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=3, weights=uniform;, score=0.847 total time=   0.4s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=3, weights=uniform;, score=0.822 total time=   0.4s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=3, weights=uniform;, score=0.824 total time=   0.4s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=3, weights=uniform;, score=0.818 total time=   0.4s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=3, weights=uniform;, score=0.838 total time=   0.4s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=3, weights=distance;, score=0.848 total time=   0.3s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=3, weights=distance;, score=0.822 total time=   0.3s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=3, weights=distance;, score=0.825 total time=   0.3s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=3, weights=distance;, score=0.819 total time=   0.3s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=3, weights=distance;, score=0.838 total time=   0.3s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=5, weights=uniform;, score=0.836 total time=   0.4s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=5, weights=uniform;, score=0.827 total time=   0.4s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=5, weights=uniform;, score=0.828 total time=   0.4s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=5, weights=uniform;, score=0.827 total time=   0.4s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=5, weights=uniform;, score=0.842 total time=   0.4s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=5, weights=distance;, score=0.839 total time=   0.3s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=5, weights=distance;, score=0.827 total time=   0.3s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=5, weights=distance;, score=0.831 total time=   0.3s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=5, weights=distance;, score=0.827 total time=   0.3s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=5, weights=distance;, score=0.841 total time=   0.3s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=7, weights=uniform;, score=0.841 total time=   0.4s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=7, weights=uniform;, score=0.829 total time=   0.4s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=7, weights=uniform;, score=0.832 total time=   0.3s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=7, weights=uniform;, score=0.824 total time=   0.4s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=7, weights=uniform;, score=0.837 total time=   0.4s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=7, weights=distance;, score=0.840 total time=   0.3s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=7, weights=distance;, score=0.826 total time=   0.3s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=7, weights=distance;, score=0.831 total time=   0.5s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=7, weights=distance;, score=0.825 total time=   0.4s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=7, weights=distance;, score=0.839 total time=   0.3s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=9, weights=uniform;, score=0.846 total time=   0.7s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=9, weights=uniform;, score=0.827 total time=   0.5s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=9, weights=uniform;, score=0.837 total time=   0.4s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=9, weights=uniform;, score=0.824 total time=   0.6s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=9, weights=uniform;, score=0.841 total time=   0.4s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=9, weights=distance;, score=0.848 total time=   0.4s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=9, weights=distance;, score=0.828 total time=   0.3s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=9, weights=distance;, score=0.835 total time=   0.3s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=9, weights=distance;, score=0.825 total time=   0.3s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=9, weights=distance;, score=0.845 total time=   0.3s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=11, weights=uniform;, score=0.842 total time=   0.4s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=11, weights=uniform;, score=0.829 total time=   0.5s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=11, weights=uniform;, score=0.842 total time=   0.5s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=11, weights=uniform;, score=0.827 total time=   0.4s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=11, weights=uniform;, score=0.837 total time=   0.4s\n",
      "[CV 1/5] END algorithm=ball_tree, n_neighbors=11, weights=distance;, score=0.844 total time=   0.3s\n",
      "[CV 2/5] END algorithm=ball_tree, n_neighbors=11, weights=distance;, score=0.832 total time=   0.3s\n",
      "[CV 3/5] END algorithm=ball_tree, n_neighbors=11, weights=distance;, score=0.839 total time=   0.3s\n",
      "[CV 4/5] END algorithm=ball_tree, n_neighbors=11, weights=distance;, score=0.832 total time=   0.3s\n",
      "[CV 5/5] END algorithm=ball_tree, n_neighbors=11, weights=distance;, score=0.840 total time=   0.3s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=KNeighborsClassifier(),\n",
       "             param_grid={'algorithm': ['auto', 'kd_tree', 'ball_tree'],\n",
       "                         'n_neighbors': [1, 3, 5, 7, 9, 11],\n",
       "                         'weights': ['uniform', 'distance']},\n",
       "             verbose=3)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model with GridSearch\n",
    "grid.fit(X_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The models best given parameters: {'algorithm': 'auto', 'n_neighbors': 11, 'weights': 'distance'} \n",
      " The models best score given best parameters: 0.8373063404623815\n"
     ]
    }
   ],
   "source": [
    "print(f\" The models best given parameters: {grid.best_params_} \")\n",
    "print(F\" The models best score given best parameters: {grid.best_score_}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save 'grid' variable as 'knn' variable\n",
    "knn = grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                precision    recall  f1-score   support\n",
      "\n",
      "     CONFIRMED       0.70      0.61      0.65       411\n",
      "FALSE POSITIVE       0.72      0.79      0.75       484\n",
      "     CANDIDATE       0.97      0.97      0.97       853\n",
      "\n",
      "      accuracy                           0.84      1748\n",
      "     macro avg       0.79      0.79      0.79      1748\n",
      "  weighted avg       0.83      0.84      0.83      1748\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make predictions\n",
    "predictions = knn.predict(X_test_scaled)\n",
    "\n",
    "#Calculate classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, predictions,\n",
    "                           target_names = [\"CONFIRMED\", \"FALSE POSITIVE\", \"CANDIDATE\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Models/KNN_model.sav']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save your model by updating \"your_name\" with your name\n",
    "# and \"your_model\" with your model variable\n",
    "# be sure to turn this in to BCS\n",
    "# if joblib fails to import, try running the command to install in terminal/git-bash\n",
    "import joblib\n",
    "filename = 'Models/KNN_model.sav'\n",
    "joblib.dump(knn, filename)"
   ]
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "dev"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "nteract": {
   "version": "0.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}