ml_demo_fhnw_python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Importing-the-packages-required-for-this-demo\" data-toc-modified-id=\"Importing-the-packages-required-for-this-demo-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Importing the packages required for this demo</a></span></li><li><span><a href=\"#Loading-and-preparing-the-example-datasets\" data-toc-modified-id=\"Loading-and-preparing-the-example-datasets-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Loading and preparing the example datasets</a></span><ul class=\"toc-item\"><li><span><a href=\"#Visual-overview-of-the-dataset\" data-toc-modified-id=\"Visual-overview-of-the-dataset-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Visual overview of the dataset</a></span></li><li><span><a href=\"#Boxplot-of-the-feature-variables\" data-toc-modified-id=\"Boxplot-of-the-feature-variables-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Boxplot of the feature variables</a></span></li><li><span><a href=\"#Looking-at-the-data-itself\" data-toc-modified-id=\"Looking-at-the-data-itself-2.3\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Looking at the data itself</a></span></li><li><span><a href=\"#Checking-for-data-types-and-missing-values\" data-toc-modified-id=\"Checking-for-data-types-and-missing-values-2.4\"><span class=\"toc-item-num\">2.4&nbsp;&nbsp;</span>Checking for data types and missing values</a></span></li><li><span><a href=\"#Looking-at-some-statistics-of-the-feature-variables\" data-toc-modified-id=\"Looking-at-some-statistics-of-the-feature-variables-2.5\"><span class=\"toc-item-num\">2.5&nbsp;&nbsp;</span>Looking at some statistics of the feature variables</a></span></li><li><span><a href=\"#Checking-the-distribution-of-the-target-variable\" data-toc-modified-id=\"Checking-the-distribution-of-the-target-variable-2.6\"><span class=\"toc-item-num\">2.6&nbsp;&nbsp;</span>Checking the distribution of the target variable</a></span></li></ul></li><li><span><a href=\"#Setting-up-the-models\" data-toc-modified-id=\"Setting-up-the-models-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Setting up the models</a></span><ul class=\"toc-item\"><li><span><a href=\"#Splitting-data-into-training-and-test-data-and-setting-cross-validtation-folds\" data-toc-modified-id=\"Splitting-data-into-training-and-test-data-and-setting-cross-validtation-folds-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Splitting data into training and test data and setting cross validtation folds</a></span></li><li><span><a href=\"#Defining-a-workflow-that-can-be-used-for-all-models\" data-toc-modified-id=\"Defining-a-workflow-that-can-be-used-for-all-models-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Defining a workflow that can be used for all models</a></span></li><li><span><a href=\"#Training-Decision-trees\" data-toc-modified-id=\"Training-Decision-trees-3.3\"><span class=\"toc-item-num\">3.3&nbsp;&nbsp;</span>Training Decision trees</a></span><ul class=\"toc-item\"><li><span><a href=\"#Using-standard-parameters-of-the-model:-tree-is-grown-fully-until-no-more-splits-are-possible\" data-toc-modified-id=\"Using-standard-parameters-of-the-model:-tree-is-grown-fully-until-no-more-splits-are-possible-3.3.1\"><span class=\"toc-item-num\">3.3.1&nbsp;&nbsp;</span>Using standard parameters of the model: tree is grown fully until no more splits are possible</a></span></li><li><span><a href=\"#Playing-with-the-tree-depth\" data-toc-modified-id=\"Playing-with-the-tree-depth-3.3.2\"><span class=\"toc-item-num\">3.3.2&nbsp;&nbsp;</span>Playing with the tree depth</a></span></li></ul></li><li><span><a href=\"#Training-Random-Forests\" data-toc-modified-id=\"Training-Random-Forests-3.4\"><span class=\"toc-item-num\">3.4&nbsp;&nbsp;</span>Training Random Forests</a></span><ul class=\"toc-item\"><li><span><a href=\"#Using-standard-parameters-of-the-model:-100-fully-grown-trees\" data-toc-modified-id=\"Using-standard-parameters-of-the-model:-100-fully-grown-trees-3.4.1\"><span class=\"toc-item-num\">3.4.1&nbsp;&nbsp;</span>Using standard parameters of the model: 100 fully grown trees</a></span></li><li><span><a href=\"#Playing-with--number-of-trees-and-maximum-depth-for-individual-trees\" data-toc-modified-id=\"Playing-with--number-of-trees-and-maximum-depth-for-individual-trees-3.4.2\"><span class=\"toc-item-num\">3.4.2&nbsp;&nbsp;</span>Playing with  number of trees and maximum depth for individual trees</a></span></li></ul></li><li><span><a href=\"#Training-K-Nearest-Neighbors\" data-toc-modified-id=\"Training-K-Nearest-Neighbors-3.5\"><span class=\"toc-item-num\">3.5&nbsp;&nbsp;</span>Training K Nearest Neighbors</a></span><ul class=\"toc-item\"><li><span><a href=\"#Using-standard-parameters-of-the-model:-5-neighbors\" data-toc-modified-id=\"Using-standard-parameters-of-the-model:-5-neighbors-3.5.1\"><span class=\"toc-item-num\">3.5.1&nbsp;&nbsp;</span>Using standard parameters of the model: 5 neighbors</a></span></li><li><span><a href=\"#Scaling-all-feature--variables-into-[0,-1]-intervals\" data-toc-modified-id=\"Scaling-all-feature--variables-into-[0,-1]-intervals-3.5.2\"><span class=\"toc-item-num\">3.5.2&nbsp;&nbsp;</span>Scaling all feature  variables into [0, 1]-intervals</a></span></li><li><span><a href=\"#Scaling-and-playing-with-the-number-of-neighbors\" data-toc-modified-id=\"Scaling-and-playing-with-the-number-of-neighbors-3.5.3\"><span class=\"toc-item-num\">3.5.3&nbsp;&nbsp;</span>Scaling and playing with the number of neighbors</a></span></li></ul></li><li><span><a href=\"#Training-Naive-Bayes\" data-toc-modified-id=\"Training-Naive-Bayes-3.6\"><span class=\"toc-item-num\">3.6&nbsp;&nbsp;</span>Training Naive Bayes</a></span><ul class=\"toc-item\"><li><span><a href=\"#Using-the-standard-model\" data-toc-modified-id=\"Using-the-standard-model-3.6.1\"><span class=\"toc-item-num\">3.6.1&nbsp;&nbsp;</span>Using the standard model</a></span></li><li><span><a href=\"#Trying-to-automatically-eliminate-feature-variables-with-only-little-predictive-power\" data-toc-modified-id=\"Trying-to-automatically-eliminate-feature-variables-with-only-little-predictive-power-3.6.2\"><span class=\"toc-item-num\">3.6.2&nbsp;&nbsp;</span>Trying to automatically eliminate feature variables with only little predictive power</a></span></li></ul></li><li><span><a href=\"#Adding-a-model-by-yourself\" data-toc-modified-id=\"Adding-a-model-by-yourself-3.7\"><span class=\"toc-item-num\">3.7&nbsp;&nbsp;</span>Adding a model by yourself</a></span></li></ul></li><li><span><a href=\"#Comparing-the-models\" data-toc-modified-id=\"Comparing-the-models-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Comparing the models</a></span><ul class=\"toc-item\"><li><span><a href=\"#Visual-comparison-of-cross-validation-scores-with-a-box-plot\" data-toc-modified-id=\"Visual-comparison-of-cross-validation-scores-with-a-box-plot-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>Visual comparison of cross validation scores with a box plot</a></span></li><li><span><a href=\"#Summary-of-cross-validation-scores\" data-toc-modified-id=\"Summary-of-cross-validation-scores-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>Summary of cross validation scores</a></span></li><li><span><a href=\"#Summary-of-overall-scores-when-predicting-test-data\" data-toc-modified-id=\"Summary-of-overall-scores-when-predicting-test-data-4.3\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>Summary of overall scores when predicting test data</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing the packages required for this demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:51:29.355095Z",
     "start_time": "2021-04-13T13:51:21.606690Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, cross_validate\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import confusion_matrix, plot_confusion_matrix, accuracy_score\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.tree import DecisionTreeClassifier, plot_tree\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "# package shap is not installed by default in colab, so we install it befor import if not available\n",
    "try:\n",
    "    import shap\n",
    "except Exception:\n",
    "    !pip install shap\n",
    "    import shap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-08T11:40:06.025739Z",
     "start_time": "2021-04-08T11:40:05.969161Z"
    }
   },
   "source": [
    "# Loading and preparing the example datasets\n",
    "\n",
    "The python package scikit-learn which carries the majority of the workload of this machine learning demo provides [several toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html). We will be using three of them that are suitable for classification purposes:\n",
    "\n",
    "**iris** classifies 150 observations of flowers into 3 categories and providing 4 feature variables, the length and width of their sepals and petals. For more info see this [Wikipedia article about the iris dataset](https://en.wikipedia.org/wiki/Iris_flower_data_set).\n",
    "\n",
    "![sepal and petal of iris](iris_petal_sepal.png)\n",
    "\n",
    "**wine** classifies 178 wines into 3 categories and provides 13 feature variables such as alcohol, color intensity, etc. \n",
    "\n",
    "**cancer** classifies 569 observations of breast cancer into 2 categories and provides 30 feature variables such as radius, texture, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:51:29.379437Z",
     "start_time": "2021-04-13T13:51:29.357186Z"
    }
   },
   "outputs": [],
   "source": [
    "iris = datasets.load_iris()\n",
    "iris_data = pd.DataFrame(data=iris.data, columns=iris.feature_names)\n",
    "iris_data['target'] = iris.target_names[iris.target]\n",
    "\n",
    "wine = datasets.load_wine()\n",
    "wine_data = pd.DataFrame(data=wine.data, columns=wine.feature_names)\n",
    "wine_data['target'] = wine.target\n",
    "\n",
    "cancer = datasets.load_breast_cancer()\n",
    "cancer_data = pd.DataFrame(data=cancer.data, columns=cancer.feature_names)\n",
    "cancer_data['target'] = cancer.target_names[cancer.target]\n",
    "\n",
    "data = {'iris': iris_data, 'wine': wine_data, 'cancer': cancer_data}\n",
    "\n",
    "selected = 'iris' # here we can change the dataset to 'iris', 'wine' or 'cancer'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-08T11:42:05.913235Z",
     "start_time": "2021-04-08T11:42:05.910574Z"
    }
   },
   "source": [
    "## Visual overview of the dataset\n",
    "By choosing the target variable as hue, we can have pair plots of all the feature variables. This overview can be a first hint towards eliminating variables that have not much prediction power.\n",
    "\n",
    "For the **wine** and especially the **cancer** dataset this visualization takes quite some time to show due to the number of feature variables. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:02.614772Z",
     "start_time": "2021-04-13T13:51:29.382120Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.show(sns.pairplot(data.get(selected), hue='target'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Boxplot of the feature variables\n",
    "Caveat: the **wine** and **cancer** datasets are plotted on a logarithmic scale as some variables are much bigger than other variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.611448Z",
     "start_time": "2021-04-13T13:52:02.617703Z"
    }
   },
   "outputs": [],
   "source": [
    "tmp = pd.melt(data.get(selected), id_vars=['target']) \n",
    "fig, ax = plt.subplots(figsize=(15, 10))\n",
    "ax.set(xscale='linear' if selected == 'iris' else 'log')\n",
    "plt.show(sns.boxplot(ax=ax, data=tmp, y='variable', x='value', hue='target'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looking at the data itself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.635712Z",
     "start_time": "2021-04-13T13:52:03.613695Z"
    }
   },
   "outputs": [],
   "source": [
    "data.get(selected).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking for data types and missing values\n",
    "all feature variables are numeric -> all good, no one-hot-encoding of categories required<br>\n",
    "no missing values -> all good, no value imputation or dropping of variables or data points required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.645040Z",
     "start_time": "2021-04-13T13:52:03.637617Z"
    }
   },
   "outputs": [],
   "source": [
    "display(data.get(selected).dtypes)\n",
    "display(data.get(selected).isnull().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looking at some statistics of the feature variables\n",
    "In the **wine** dataset we see that the values of one variable -- proline -- are much bigger than the values of all other variables. This can cause some issues with some models that rely on e.g. euclidean distances. In that case normalizing all variables to [0,1]-intervals would be a sensible preprocessing step. Same applies to the **cancer** dataset. For the **iris** dataset all is fine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.695895Z",
     "start_time": "2021-04-13T13:52:03.646536Z"
    }
   },
   "outputs": [],
   "source": [
    "data.get(selected).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking the distribution of the target variable\n",
    "The **iris** data has a pefectly balanced distribution, the other two data sets are not heavily imbalanced -> that's ok, we don't have to take measures, can use simple metrics such as accuracy, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.706125Z",
     "start_time": "2021-04-13T13:52:03.700468Z"
    }
   },
   "outputs": [],
   "source": [
    "data.get(selected).target.value_counts()/data.get(selected).shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting up the models"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting data into training and test data and setting cross validtation folds\n",
    "We make a split of 85% training and 15% test data. Then we use use 5-fold [cross validation](https://en.wikipedia.org/wiki/Cross-validation_(statistics)) on the training data (i.e. splitting data into 5 blocks and making 5 training runs using 4 of these blocks, validating with the 5th) for calibration of paramters. \n",
    "\n",
    "![cross validation](crossval.png)\n",
    "\n",
    "This helps us to find a good tradeoff between bias and variance in order to to avoid overfitting (performing *too well* on the training data because a model captured all its noise) or other errors due to having selected a particular data split.\n",
    "\n",
    "![fitting](fitting.png)\n",
    "\n",
    "Finally we can fit a model on the entire training data and test its predictions on the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.716989Z",
     "start_time": "2021-04-13T13:52:03.710497Z"
    }
   },
   "outputs": [],
   "source": [
    "tr_size = 0.85 # fraction of data used for training, paramter for experimenting\n",
    "cv_runs = 5 # number of runs of cross validation, parameter for experimenting\n",
    "\n",
    "np.random.seed(12345)\n",
    "train, test = train_test_split(data.get(selected), train_size=tr_size,\n",
    "                               stratify=data.get(selected).target) \n",
    "\n",
    "train_feat = train.iloc[:, :-1] # selection of feature variables (all but the last variable)\n",
    "train_tgt = train.iloc[:, -1:] # selection of target variable (last variable)\n",
    "\n",
    "test_feat = test.iloc[:, :-1] # selection of feature variables (all but the last variable)\n",
    "test_tgt = test.iloc[:, -1:] # selection of target variable (last variable)\n",
    "\n",
    "models = {} # saving models in this dictionary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining a workflow that can be used for all models\n",
    "As we perform the same steps for every model, we can put these steps into a function that can be called each time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:03.724536Z",
     "start_time": "2021-04-13T13:52:03.718813Z"
    }
   },
   "outputs": [],
   "source": [
    "def eval_model(model, train_feat, train_tgt, test_feat, test_tgt, cv_runs=5, disp=True, plot=True):\n",
    "    # ensuring that each partitioning is made the same way, makes results reproducible\n",
    "    np.random.seed(12345)\n",
    "    # Cross-Validating the classifier on the training data\n",
    "    cv_score = cross_val_score(model, train_feat.values, train_tgt.values.ravel(), cv=cv_runs,\n",
    "                              scoring='accuracy')\n",
    "    # Training the model on the training data\n",
    "    model.fit(train_feat.values, train_tgt.values.ravel())\n",
    "    # Evaluating the model on the test data\n",
    "    score = model.score(test_feat.values, test_tgt.values.ravel())\n",
    "    # Creating a confusion matrix for the test data\n",
    "    conf_mat = confusion_matrix(test_tgt.values.ravel(),\n",
    "                                model.predict(test_feat.values), labels=model.classes_)\n",
    "    result = {'model': model, 'cv_score': cv_score, 'score': score}\n",
    "    if disp:\n",
    "        display(result)\n",
    "    if plot:\n",
    "        fig, ax = plt.subplots(1, 2, figsize=(15, 3))\n",
    "        # boxplot of cross validation results\n",
    "        bp = sns.boxplot(data=cv_score, ax=ax[0])\n",
    "        bp.set(xticklabels=[])\n",
    "        bp.set_title('Cross Validation')\n",
    "        # confusion matrix for test data\n",
    "        cm = plot_confusion_matrix(model, test_feat.values, test_tgt.values.ravel(), ax=ax[1]) \n",
    "    result.update({'conf_mat': conf_mat})\n",
    "    return result\n",
    "\n",
    "def shap_summary(model, train_feat, test_feat, tree_explainer=False):\n",
    "    # creating a shapley plot to find out variable importance\n",
    "    exp = (shap.TreeExplainer(model) if tree_explainer\n",
    "           else shap.KernelExplainer(model.predict_proba, train_feat))\n",
    "    shap_values = exp.shap_values(test_feat)\n",
    "    ssum = shap.summary_plot(shap_values, test_feat, plot_type='bar',\n",
    "                             class_names=model.classes_, plot_size=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Decision trees\n",
    "These algorithms consist of a series of comparisons of the values of feature variables with limits determined during the training process. For details see the corresponding [wikipedia article](https://en.wikipedia.org/wiki/Decision_tree_learning).\n",
    "\n",
    "![cross validation](decisiontree.png)\n",
    "\n",
    "### Using standard parameters of the model: tree is grown fully until no more splits are possible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:04.814806Z",
     "start_time": "2021-04-13T13:52:03.726326Z"
    }
   },
   "outputs": [],
   "source": [
    "models.update({'tree_std':\n",
    "               eval_model(DecisionTreeClassifier(),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('tree_std').get('model'), train_feat, test_feat, tree_explainer=True)\n",
    "\n",
    "# This plot is only available for decision trees\n",
    "fig, ax = plt.subplots(figsize=(15, 8))\n",
    "plt.show(plot_tree(models.get('tree_std').get('model'), filled=True, ax=ax))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Playing with the tree depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:05.829567Z",
     "start_time": "2021-04-13T13:52:04.817138Z"
    }
   },
   "outputs": [],
   "source": [
    "tree_depth = 2 # maximum depth of tree, parameter for experimenting\n",
    "\n",
    "models.update({'tree':\n",
    "               eval_model(DecisionTreeClassifier(max_depth=tree_depth),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('tree').get('model'), train_feat, test_feat, tree_explainer=True)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(15, 8))\n",
    "plt.show(plot_tree(models.get('tree').get('model'), filled=True, ax=ax))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Random Forests\n",
    "\n",
    "These so-called ensemble algorithms predict by using a majority vote of simpler models such as classification trees. For details see the corresponding [Wikipedia article](https://en.wikipedia.org/wiki/Random_forest).\n",
    "\n",
    "![Random Forest](randomforest.png)\n",
    "\n",
    "### Using standard parameters of the model: 100 fully grown trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:06.816155Z",
     "start_time": "2021-04-13T13:52:05.831832Z"
    }
   },
   "outputs": [],
   "source": [
    "models.update({'rfor_std':\n",
    "               eval_model(RandomForestClassifier(),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('rfor_std').get('model'), train_feat, test_feat, tree_explainer=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Playing with  number of trees and maximum depth for individual trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:07.278430Z",
     "start_time": "2021-04-13T13:52:06.818891Z"
    }
   },
   "outputs": [],
   "source": [
    "n_trees = 5 # number of trees, parameter for experimenting\n",
    "rfor_depth = 2 # maximum depth of individual trees, parameter for experimenting\n",
    "\n",
    "models.update({'rfor':\n",
    "               eval_model(RandomForestClassifier(n_estimators=n_trees, max_depth=rfor_depth),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('rfor').get('model'), train_feat, test_feat, tree_explainer=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training K Nearest Neighbors\n",
    "\n",
    "These nonparametric algorithms classify by a majority vote of those data points closest to the data point one wants to classify. For details see the corresponding [Wikipedia article](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm). \n",
    "![k nearest neighbors](knn.png)\n",
    "\n",
    "### Using standard parameters of the model: 5 neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:07.502248Z",
     "start_time": "2021-04-13T13:52:07.280355Z"
    }
   },
   "outputs": [],
   "source": [
    "models.update({'knn_std':\n",
    "               eval_model(KNeighborsClassifier(),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Shapley plot is separated here because for Non-Decision-Tree-Models it takes quite long to compute, when using the **wine** and especially the **cancer** data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:41.848634Z",
     "start_time": "2021-04-13T13:52:07.504591Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('knn_std').get('model'), train_feat, test_feat, tree_explainer=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaling all feature  variables into [0, 1]-intervals\n",
    "As these algorithms rely on euclidean distances, they are sensitive towards different scales of feature variables, so e.g. normalizing them to [0, 1]-intervals is recommended."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:52:42.086253Z",
     "start_time": "2021-04-13T13:52:41.850391Z"
    }
   },
   "outputs": [],
   "source": [
    "# Here a pipeline is set up: scale feature variables to [0, 1]-intervals, then apply model\n",
    "models.update({'knn_scl':\n",
    "               eval_model(make_pipeline(MinMaxScaler(), KNeighborsClassifier()),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:53:31.581170Z",
     "start_time": "2021-04-13T13:52:42.088317Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('knn_scl').get('model'), train_feat, test_feat, tree_explainer=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaling and playing with the number of neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:53:31.842454Z",
     "start_time": "2021-04-13T13:53:31.583563Z"
    }
   },
   "outputs": [],
   "source": [
    "neighbors = 5 # number of neighbors used, parameter for experimenting\n",
    "\n",
    "models.update({'knn':\n",
    "               eval_model(make_pipeline(MinMaxScaler(), KNeighborsClassifier(n_neighbors=neighbors)),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:54:20.611056Z",
     "start_time": "2021-04-13T13:53:31.844241Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('knn').get('model'), train_feat, test_feat, tree_explainer=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Naive Bayes\n",
    "\n",
    "This model is based on Bayes' theorem and uses conditional probabilities. Each data point is assigned to the class with the highest\n",
    "\\[ P(class|data) = \\frac{P(data|class) \\times P(class)}{P(data)} \\nonumber \\]\n",
    "For details see the corresponding [Wikipedia article](https://en.wikipedia.org/wiki/Naive_Bayes_classifier).\n",
    "\n",
    "### Using the standard model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:54:20.881199Z",
     "start_time": "2021-04-13T13:54:20.613929Z"
    }
   },
   "outputs": [],
   "source": [
    "models.update({'bayes':\n",
    "               eval_model(GaussianNB(),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:54:49.821613Z",
     "start_time": "2021-04-13T13:54:20.883075Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('bayes').get('model'), train_feat, test_feat, tree_explainer=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying to automatically eliminate feature variables with only little predictive power\n",
    "Here we try to keep only the k feature variables with the highes predictive power according to the ANOVA F Value, for details see the corresponding [Wikipedia article](https://en.wikipedia.org/wiki/F-test)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:57:36.792112Z",
     "start_time": "2021-04-13T13:57:36.539129Z"
    }
   },
   "outputs": [],
   "source": [
    "features = 2 # number of feature variables to be used, parameter for experimenting\n",
    "\n",
    "# Here a pipeline is set up: find the k feature variables with most predictive power, then apply model\n",
    "models.update({'bayes_sel':\n",
    "               eval_model(make_pipeline(SelectKBest(f_classif, k=features), GaussianNB()),\n",
    "                          train_feat, train_tgt, test_feat, test_tgt, cv_runs)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:58:06.245760Z",
     "start_time": "2021-04-13T13:57:40.090073Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10, 3))\n",
    "shap_summary(models.get('bayes_sel').get('model'), train_feat, test_feat, tree_explainer=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding a model by yourself\n",
    "Gradient boosting is an ensemble technique where a model is trained in an iterative way where the weights of the individual data points change from iteration to iteration in order to improve the prediction.\n",
    "\n",
    "![gradient boosting](gradboost.png)\n",
    "\n",
    "Go to the [sklearn's api description](https://scikit-learn.org/stable/modules/classes.html), look up the Gradient Boosting Classifier (hint: it's an ensemble algorithm) and try to find out which are the main parameters (hint: how fast should it learn? how many iterations?) for setting it up. Then create another model using\n",
    "\n",
    "*from* **[which package do you need?]** *import * **[Which Classifier do you need?]**<br>\n",
    "*models.update({'grd_bst': eval_model(* **[your model with parameters to play with goes here]** *, train_feat, train_tgt, test_feat, test_tgt, cv_runs)})*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-09T12:10:25.807396Z",
     "start_time": "2021-04-09T12:10:24.241263Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing the models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-08T13:15:24.615360Z",
     "start_time": "2021-04-08T13:15:24.613563Z"
    }
   },
   "source": [
    "## Visual comparison of cross validation scores with a box plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:55:16.374124Z",
     "start_time": "2021-04-13T13:55:16.148888Z"
    }
   },
   "outputs": [],
   "source": [
    "cv_data = pd.DataFrame({k: v.get('cv_score') for k, v in models.items()})\n",
    "fig, ax = plt.subplots(figsize=(15, 6))\n",
    "plt.show(sns.boxplot(ax=ax, data=cv_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-08T13:16:08.099609Z",
     "start_time": "2021-04-08T13:16:08.097559Z"
    }
   },
   "source": [
    "## Summary of cross validation scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-13T13:55:16.419494Z",
     "start_time": "2021-04-13T13:55:16.392267Z"
    }
   },
   "outputs": [],
   "source": [
    "cv_data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary of overall scores when predicting test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-26T07:35:56.426282Z",
     "start_time": "2021-04-26T07:35:55.025576Z"
    }
   },
   "outputs": [],
   "source": [
    "res_data =  pd.DataFrame({k: v.get('score') for k, v in models.items()}, index=[0]).melt()\n",
    "fig, ax = plt.subplots(figsize=(15, 6))\n",
    "plt.show(sns.barplot(ax=ax, data=res_data,  x='variable', y='value'))"
   ]
  }
 ],
 "metadata": {
  "author": "me",
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}