Partie 2_Noisy Data_Outliers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 474,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "import scipy.stats as ss\n",
    "from sklearn.base import BaseEstimator\n",
    "import inspect\n",
    "from collections import Counter\n",
    "from sklearn.metrics import make_scorer,recall_score,roc_auc_score,plot_confusion_matrix,f1_score\n",
    "from sklearn.utils import resample\n",
    "from sklearn.model_selection import train_test_split,StratifiedShuffleSplit\n",
    "from sklearn.linear_model import LogisticRegression,SGDClassifier,HuberRegressor,RANSACRegressor,TheilSenRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier,BaggingClassifier,AdaBoostClassifier,IsolationForest,ExtraTreesClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC,OneClassSVM\n",
    "from sklearn.model_selection import RepeatedKFold,RepeatedStratifiedKFold,cross_val_score,cross_validate,GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.cluster import MeanShift\n",
    "from imblearn.over_sampling import RandomOverSampler,SMOTE, ADASYN\n",
    "from imblearn.under_sampling import RandomUnderSampler,NearMiss,TomekLinks,ClusterCentroids,EditedNearestNeighbours,NeighbourhoodCleaningRule\n",
    "from imblearn.datasets import make_imbalance\n",
    "from imblearn.pipeline import Pipeline\n",
    "from sklearn.base import BaseEstimator,clone\n",
    "from sklearn.multiclass import OneVsRestClassifier,OneVsOneClassifier\n",
    "from sklearn.preprocessing import RobustScaler,StandardScaler,LabelEncoder\n",
    "import time\n",
    "from cleanlab.classification import LearningWithNoisyLabels\n",
    "from cleanlab.noise_generation import generate_noise_matrix_from_trace\n",
    "from cleanlab.noise_generation import generate_noisy_labels\n",
    "from cleanlab.util import print_noise_matrix\n",
    "import copy\n",
    "from skclean.models import RobustLR,RobustForest\n",
    "from skclean.simulate_noise import UniformNoise\n",
    "from sklearn.neighbors import KNeighborsClassifier,DistanceMetric\n",
    "from sklearn.svm import SVC\n",
    "import eif as eif\n",
    "from sklearn.impute import SimpleImputer\n",
    "from skclean.detectors import MCS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyod.models.pca import PCA\n",
    "from pyod.models.mcd import MCD\n",
    "from pyod.models.ocsvm import OCSVM\n",
    "from pyod.models.lof import LOF\n",
    "from pyod.models.cblof import CBLOF\n",
    "from pyod.models.knn import KNN\n",
    "from pyod.models.hbos import HBOS\n",
    "from pyod.models.abod import ABOD\n",
    "from pyod.models.feature_bagging import FeatureBagging\n",
    "from pyod.models.lscp import LSCP\n",
    "from pyod.models.lmdd import LMDD\n",
    "from pyod.models.copod import COPOD\n",
    "from pyod.models.sos import SOS\n",
    "from pyod.models.sod import SOD\n",
    "from pyod.models.so_gaal import SO_GAAL\n",
    "from pyod.models.mo_gaal import MO_GAAL\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importation Données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>A3</th>\n",
       "      <th>A4</th>\n",
       "      <th>A5</th>\n",
       "      <th>A6</th>\n",
       "      <th>A7</th>\n",
       "      <th>A8</th>\n",
       "      <th>A9</th>\n",
       "      <th>A10</th>\n",
       "      <th>S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   A1  A2  A3  A4  A5  A6  A7  A8  A9  A10  S\n",
       "0  16   5   1   1   1   2   1   3   1    1  2\n",
       "1  41   5   4   4   5   7  10   3   2    1  2\n",
       "2  15   3   1   1   1   2   2   3   1    1  2\n",
       "3  41   6   8   8   1   3   4   3   7    1  2\n",
       "4  17   4   1   1   3   2   1   3   1    1  2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('wisconsin.csv',sep=',')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Format Features:(663, 10) ; Format Labels:(663,)\n"
     ]
    }
   ],
   "source": [
    "# Division en Features & Labels - Versions sans bruit\n",
    "dfVal = df.values\n",
    "X_clean, y_clean = dfVal[:, :-1], dfVal[:, -1]\n",
    "print('Format Features:{} ; Format Labels:{}'.format(X_clean.shape,y_clean.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline - Clean Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.9679 ; Training Accuracy: 0.9713 ; Roc_Auc: 0.9638 \n"
     ]
    }
   ],
   "source": [
    "# On instancie l'objet cross-validation avec stratification et répétition\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# On instancie le modèle\n",
    "cv_clean_y = cross_validate(LogisticRegression(random_state=1), X_clean, y_clean, scoring={'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                            cv=cv, n_jobs=-1,return_estimator=True,return_train_score=True)\n",
    "# On récupère les scores au pluriel puisque nous validons sur plusieurs vagues et on moyenne sur ces différents scores\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f ' % (np.mean(cv_clean_y['test_accuracy']),\n",
    "                                                               np.mean(cv_clean_y['train_accuracy']),\n",
    "                                                               np.mean(cv_clean_y['test_roc_auc'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Métrique utilisé - ELA"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAANYAAABRCAYAAACuVlhOAAAR40lEQVR4Ae2dC1BTZ9rHk5BE8VJQq6KRrdou9QZV0XrXYr2tVaOgdNWqtZbVDrVVuls+h4pasFWxXkCl7Sqorbgf9b5VO60fyqL1xoiLYHW8VQfFdqRxxAxkkszvmxAuSQiYSBIRX2bOcJLzvLffef857/ue5zlHgvgTBAQBlxOQuDxHkaEgIAgghCU6gSDgBgJCWG6AKrIUBISwRB8QBNxAQAjLDVBFloKAEJboA4KAGwgIYbkBqshSEBDCEn1AEHADASEsN0AVWQoCQliiDwgCbiAghOUGqCJLQUAIS/QBQcANBISw3ABVZCkICGHV4z5QevMnVr/3Ed9cM9Rcy9IbHNkQzZzwSYROm0dsygkKazJ3xrbmEt105D4Z8eGMVcdyuNhNRXgwWyEsD8J2uKiS6xxaHk53HxlSRV/iL9agFO1ZVoa0oUXwe2z+MZPDyXN4pXkj/jQ5lau2SZyxdbiirjMszY4luLEUiXIoX9wwui7jJ5STENYTAm+/WC1X9i9lUs+uDJ4yhf5tvJAo+vP5ZVuVmFLruRDfD+9GwSzL1ZdnV8qp6O4oZG2YkvYbVd3TGVv7NXPrt4brbBrTga7dOiJX9CQ2p6I9bi3VrZkLYbkVr5OZG+9yYMVS/pX3AHjItgmNkCgGsOKKHWHp/sOCP8tR9F7Cfy36oe7Ux3SVS2k+PoXfK4p3xrYijcf+G7m3dxadeywkfeVrKL06M/+YzmOlu6sgISx3ka1zvlq2q2sWlj53Cb0VUnzfTEdrWVbxDkKbS/FSRXC41HzAGVvLrDyyrz1BdJA/4WmF3E0ehVLWmpn7SjxStDsLEcJyJ9065V27sLTpb+Ir9aLz/GNY/b7rslj4ZznSRqP5qvyS5YxtnarsdGIDl9aE4DdoBXl6KN6mprHUG/W2p3/1QgjL6c7gqQS1C+s306+7RE636FPWwtLnsPgVBRLFYBLKVxOdsfVU60zlGAvTCFcFEHnELKSSXdNoIVMSsv6WxfzQkzVyXVlCWK5j6eKcahOWkZtrh6GUKOi+6AwWUyww5BPf1ySsgawsm5s5Y2uvCUY0V05x7OhRjjq0ZXLupiNDuWIy5r9MW3UKt8pXWUoPR6DyUhC87IJ1m+xVq55/J4RVb09Q7cKquAoFLMyyuWLlsrS3AonydTbcMfVYI47b2oNRSsb7nfCSSJA4tCno//kl7Cy3WGWuz11O/2btGBeXRnp6etm2c/kE2nvJqdYmq5R2PpRc5ft1sUQv+oy0HI311U6bz6Gdmdx8VIXsZFuXr4Sw6kLPrWlrExZov5uKr0xG27cPYHV9MM2xAuTInp/B3vIDztjabZJBj16nQ+fQpsdQtc5vNzuMt9iibk3LHqMJmzyZyeVb2JggWspk+L3zvXWb7OdS/q2WY1H9UCdfR3t5LSMCZrLnj4oEpWQvHUjwR5k8rPjKQ/+FsDwE2vliahdWxUqfcngSty07cvFOJvtIUQ5aVXmT2Blb5+vpfArNwQg6+QxmZb7VIBbDtQQGKaQ0Dd3huBB0x4nqPsQ87DVcJL7/i8z7ybwcari0jpFBczhQ5Hwd65pCCKuuBN2Wvpit403L7f35/JKdcYzp3tRLcmRtZrDHYhFNl7WQALmSvnEW8xRnbN3WnvKMS04T06sJqtkHMN2ts/wz3tnA60oJph+LslFs2UEjRZnxTAzqgKrfIo6aEj04xeqJXfDrvpCM+/uZpQphnWmiZiwgcbjKvFxvvMXW0ECmbC+wHhpaFujGfSEsN8KtU9aa/2P+y3IkXire2vW7nc5h4OLqYTzn5cuopCvmyb7xLt9N74BCNY3/vWt5GXPGtk61fkRiLdlxA2kubcTYzZXjtco0utPRdJNL8Oo8n6Pl9+BMYvluVTLn7uzirfZ+zNh9lfR3RzM7+VuSNmVQVHKSjwN7EZOtB30uS14N5pNzOu7te4egUeu5rC+hMO80Z68UeXRBRAir8rTWj52iPR8wJLgrHXwUSMsXC6QKX/4U2I+QOSlYOWHor/GvucG0bNqBgVNmM31EAK07jWPF8aLqQnTG1i0oNGQsHYZKIUUikdK0y1RSLIaC+vOJjO/UxNxmmQ89I/daD3H5g61qH1SBQ3kzMY8K3YGO7Pgh9Jn1FftT5hIyYQMXNUdZ0Hsgy7ILObRgNOFxm0mY2o+RCTnWCz1uaac5UyEsN8L1TNZGHlw7yQ/79vDvzDx+s7pbbFsDZ2xt0z7pz3qyYwJpHBjDWeupWZn7168nD3Hgp/MU6krJXjKA3lHHeFCwkdE9PsDkIaW/sIxXA+aX7XuiJUJYnqAsyqgzAcO1rczo60+rNjPYa+XDZZ21acFiROAc9hcZ0WV+SJcB8ZQFB2i28IbvJL6tJa11TnX75BJhGW4cYUtSIomJNW1JbDqQX1VTww2ObEmqtE/66nDlClaVUc179zPiCR+rJrYhBO7U3ExxBCNFuT9z4doRokdOZO2Jzah9A4jKKkFz5geyCi3nkaZbdrdIDe1B2DbzgoVpIadr8GLKnOWLvmZsu2nstro34T7ELhGW8XYWCRP98ZLI8O01mfciI4ks2+Yxfag/SqmcFz/IrGqF8TZZCRPx95IglXckdP1JfrNhVGVss1eaTWxwY6QSJUO/uFF9LmFjLj4+zQTusyOsBU1a9SRidwFG421S1K3wCRhAyF/Xcs5KJCYv+dkEjlxH5SLqvW8I6xjK9ntQmhVFr1GJ3LCzwOoOQi4RFhi4vnoICqk3ozbdse7sv/+TsU19mPStxqr+huurGaKQohywArvhRlbWFR8MXN80hg5du9FRrqBnbI5HV3oqaiH+e4qAjrv5p8i5aXF7t/hXcs78wu+28yzjbfbERPLlectJZin5qfMImzaPd2dG8vV5D40DwVWv8SnlcIQKL0UQn5iWPS3/dKdJCI8g1Sa83OwXZseJ1DKtzb7x3l5mde7BwvSVvKa049ltYy8+CgJPioBrrliGPD7to0DW9m0OOPSjYCDv0z4oZM/z1h6LX6NaKWg5ER2Ef3gahXeTGaWU0XrmPidcX2rNXBwUBFxKwDXCKkplfDMp3qOSqZhPGn7dwftTV3HS8spcWfUiUsc3Q9poOEkFjk2uDJfWEOI3iBXmwB3UjaV4q7dh4XRQmbtHdgy3OLplHWvWrHFsW7+TMxrH2uqR+otC3ErAJcIq/XEu/l4yWg6cw+LYWGIXvce4l5vTeHgSdnVT+iNz/b2Qd4vmlF3h2bTZWEhauIqAyCNmIZXsYloLGcqQ9ZUhBzYp3P9Rd5KlIQG89NJLjm1dJvOlp2bO7m+9KOERBFwgrIphnS+vvr2YWJOwoqcT7KsgMOas3cUFQ96n9FHIeH76boecLYsz5vNyWzUpVYE7RKi8UAQv44LNlO4R7a23hx0LyXA0dEPYPYqnuzuCC4RVMawLIbGi4xsL2TjyBd753mo9tLItRanjaSZt5FikqD6X5f2b0W5cHGnlcTvpO5czob0X8oCFZDlyxassuYSr368jNnoRn6XlYD0y05J/aCeZng7cKa/bozqCOO7aH4vKLuGmnboLq4ZhndFgsF52r2xAKT/O9cdL3pV/2J+AVVqagvRubVHTumUPRodVxe1MDhtDUEsZMr93qEG7FnlU7WqPRdFPncx17WXWjghgZlXgDqXZSxkY/BGZjq6lYKDkwX00Go1j2/2H6MQUq+pkNPC9OgvLkB9nHta9tafasK7kwm427su3dnw05BNnWkFsNY1dNp34j4OLmBRzuGpBQnOQiE4+DF6Zbz2kNFwjYZACadNQdtjkUfP50nE8qjtDVl7BgIGL8f15cd5PZmdOwyXWjQxijjOBO6UZvN/Jy8GoWknN4R81V1gceYoJ1FlYRakTyoZ1w5Ns4l70F1k9rAU9F9vcxC1KZUIzKY1C1nHT8hdcd4HPBvjQb/kv5WHdJZyO6UUT1WwOVA/cYcPrSiTK4SRVBe5gLMokfmIQHVT9WGQO3OHU6ol08evOwoz77J+lImSd6UElRgoSh6MqW643cmtrKIFTtttfaKnp5Brvc+3scbKyshzbTvyXAvsj45pKEN8/xQTqKCwNu6a3RSbvzv+ctpjslFxn/4I+PCf3r4zmrGCk2TWdtjI5Xf7+c9WVTHuFPfODaaYI4KMT5ny02XEMbC6l0djNVIvc0Z0mupspVqkz86sCdyj4bhXJ5+6w6632+M3YzdX0dxk9O5lvkzaRUVTCyY8D6RWTjR49uUteJfiTc+ju7eOdoFGsv6ynpDCP02evUNRAFkQqmIv/nidQB2E94HjCX+iklCKRKvBu/hzPPWfemihlZXE1Ut/JpN2vatSD4wn8pZMSqUSKwrt5uX1zvBVme5nfbP6tBU3GUoapzPFI0qZdmJpiMRTUnydxfCeaSE2TWRk+PSPZaxWbDn9sVeOjCmTom4nkVQXuoMuOZ0ifWXy1P4W5IRPYcFHD0QW9Gbgsm8JDCxgdHsfmhKn0G5lAjsXvRFULnqG9+xnEh49FHWsxNH+Gml/XptZBWHUt2n3p9dkxBDYOJKZ64A4Pfz3JoQM/cb5QR2n2Egb0juLYgwI2ju7BB+bAHZa9GtAgHnP8+IRLyY4NprFUgnLoFzSAdxQ8PorHTNnwhGW4xtYZffFv1YYZtQfusG5EIHP2F2HUZfJhlwHlb/XQsOUNXyZ5KnDnMU+cO5MZrm9iTIeudOsoR9Ez1hx24c4CG2DeDUZYxqJcfr5wjSPRI5m49gSb1b4ERGVRojnDD1mFNkv/Rm6lhtIjbJt5wcL0yLCuwSw2B+7w9dh2TPNU4E5961TGe+yd1ZkeC9NZ+Zqy7PkTDeAdBR6n3GCEdX9HGC2atKJnxG4KjEZup6hp5RPAgJC/stY6cAeTl/zswJGsqwrc4ZuwjoSaA3eI6jWKxGfU/Uh7Ipog/3DSCu+SPEqJrPVMGsA7CoSwHpeA7m4+p3JuWtxLK+bXnDP8Uj1wh9t7Yoj88nzVqiRQmp/KvLBpzHt3JpFfn7d+g8fjVuppS2e4xJoQPwatyENPMdvUjZF6q2kA7yjw+JloMFcsj5NrcAUaKUwLRxUQifkdBSXsmtYCmTKE9RWuag2uze5rkBCW+9g+XTkXZzD/5baoUyre9FERvBrMsobi6ezBMyKE5UHY9bcoPbnL+9Os3Tji0swvKEhP38nyCe3xkgew0ClP5/rr6OxJ/kJYnqRdT8sy3tqCunVLeowOq3xBweTJYYwJaolM5lc9SsFwh8yNUcwIncSUObHsOF/lBeBaR+d6CsyBaglhOQCpYZtoOBjRCZ/BK7F4MC2mBwRdSxiEQtqUUCtP52Iy/9GTjupkLj7U8kvyBNq1fYOvrpoef+RiR+enGLwQ1lN88lxR9ZLTMfRqomJ2dU9n7mx4HaVEyfCkqidvGe+mMMFXxZyD5R7FJaZocCVBMdlAiWsdnV3RwCeUhxDWEwJfL4rVZhM3sDnSRmOp/o4CHaejuyGXmJ6GdbTyWemlh+bQvlHfci8V00My75RFGniP21J2xRKOzuYzK4RVL3r4E6iEJoOlw1QoTM7M0qZ0mZpiMRTUcz5xPJ2amF5gIEHm05PIvbfLvFcebp+It/I182tzyqqt5dtJ3mU+haaPwtHZfC6FsJ5An36ai9TuCKWpcghfVHrmPmT7RG+UIzZWNks4OrvsgZ2VTMVOAyeg+8+HvKTsyscVj9cyXGf1kMZ0+NsP1Vtuisx+Rh2dxRWrencQ39RG4OEh3vVvwuBVl8sivY2/bWdSqxeIOGT7hMdn29FZCKu2TiSO2SFg4OZ3f+OVF4KZHh3D3OFBDPr7wcoHtVYkeNYdnYWwKnqC+O8UAYPmOud+PkluwUObkBxTNsZn3tFZCMup7iSMBQHHCAhhOcZJWAkCThEQwnIKlzAWBBwjIITlGCdhJQg4RUAIyylcwlgQcIyAEJZjnISVIOAUASEsp3AJY0HAMQJCWI5xElaCgFMEhLCcwiWMBQHHCAhhOcZJWAkCThEQwnIKlzAWBBwjIITlGCdhJQg4RUAIyylcwlgQcIyAEJZjnISVIOAUASEsp3AJY0HAMQJCWI5xElaCgFMEhLCcwiWMBQHHCAhhOcZJWAkCThEQwnIKlzAWBBwj8P9zINtdyw6LUQAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L'<b>Equalized Loss of Accuracy</b> est proposée par Sáez et al. et combine la robustesse et un facteur dépendant de la précision initiale. Cette mesure tente de minimiser les problèmes liés à l'examen individuel des mesures de performance et de robustesse et peut être utilisée pour comparer facilement différents classificateurs [1].  La mesure de l'ELA est :<br>\n",
    "![image.png](attachment:image.png) <br>\n",
    "où $A_{x\\%}$ est l'accuracy au niveau de bruit $x\\%$ et $A_{0\\%}$ est l'accuracy au niveau sans bruit. <br>\n",
    "\n",
    "\n",
    "\n",
    "[1]: J. A. Sáez, J. Luengo, and F. Herrera, “Evaluating the classifier behavior with noisy data considering performance\n",
    "and robustness: The Equalized Loss of Accuracy measure”ELSEVIER, Neurocomputing 176 (2016) 26-35. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Implémentation\n",
    "def ELA(est):\n",
    "    return (1-np.mean(est['test_accuracy']))/np.mean(cv_clean_y['test_accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction de bruit uniforme"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Corruption/erreur représentée dans les valeurs d'attribut des instances dans un ensemble de données est appelé bruit d'attribut: les valeurs d'attribut erronées,manquantes, inconnues ou incomplètes. Des efforts de recherche ont été entrepris pour traiter du bruit d'attribut et l'élimination est l'un des moyens couramment utilisés.<br>\n",
    "Toutefois, leurs résultats montrent que l'élimination des cas contenant le bruit d'attribut n'est pas une bonne idée, car pouvant contenir de l'information . Par conséquent, le traitement du bruit d'attributest plus difficile et la recherche dans ce domaine n'a pas beaucoup de progrès, à l'exception de quelques efforts<br> \n",
    "On essaiera de traiter ce cas par la suite"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Afin de pouvoir contrôler le niveau de bruit dans l'ensemble de données, nous avons ajouté manuellement x% de bruit dans l'ensemble de données en changeant aléatoirement les labels de classe d'exactement x% des instances par d'autres classes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({2: 410, 4: 253})\n"
     ]
    }
   ],
   "source": [
    "X_dirty,y_dirty=UniformNoise(noise_level=0.2 # Bruit 20%\n",
    "                             ,random_state=1).fit_transform(X_clean.copy(),y_clean.copy())\n",
    "print(Counter(y_dirty))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({2: 434, 4: 229})\n"
     ]
    }
   ],
   "source": [
    "print(Counter(y_clean))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7667 ; Training Accuracy: 0.7731 ; Roc_Auc: 0.7362, ELA: 0.2411\n"
     ]
    }
   ],
   "source": [
    "# On instancie l'objet cross-validation avec stratification et répétition\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "etapes = [('StandardScaling',  StandardScaler()),\n",
    "    ('modele',LogisticRegression(C=0.1,random_state=1,solver='saga',penalty='l1'))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "# On instancie le modèle\n",
    "cv_logreg_y = cross_validate(pipeline, X_dirty, y_dirty, scoring={'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                            cv=cv, n_jobs=-1,return_estimator=True,return_train_score=True)\n",
    "# On récupère les scores au pluriel puisque nous validons sur plusieurs vagues et on moyenne sur ces différents scores\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_logreg_y['test_accuracy']),\n",
    "                                                               np.mean(cv_logreg_y['train_accuracy']),\n",
    "                                                               np.mean(cv_logreg_y['test_roc_auc']),\n",
    "                                                               ELA(cv_logreg_y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apprentissage supervisé\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Algorithmes robustes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remaque : Robuste signifie résistant aux outliers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  MOM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "De nombreux algorithmes d'apprentissage sont basés sur un paradigme connu sous le nom de minimisation du risque empirique/empirical risk minimization (ERM)  qui consiste à trouver l'estimateur $\\widehat{f}$ qui minimise une estimation du risque.<br>\n",
    "<h3><center> $\\widehat{f} = \\text{argmin}_{f\\in F}\\frac{1}{n}\\sum_{i=1}^n\\ell(f(X_i),y_i),$</center></h3>\n",
    "où le $\\ell$ est une fonction de perte (par exemple la distance au carré dans les problèmes de régression). Autrement dit, nous essayons de minimiser une estimation du risque attendu et cette estimation correspond à une moyenne empirique. Cependant, il est bien connu que la moyenne empirique n'est pas robuste aux données extrêmes et ces valeurs extrêmes auront une grande influence sur l'estimation de $\\widehat{f}$  . Le principe de l'algorithme de pondération robuste est de s'appuyer sur un estimateur robuste (tel que la Mean-of-Medians (MOM) ou le Huber Estimator au lieu de la moyenne empirique dans l'équation ci-dessus <br>\n",
    "En pratique, on peut définir des poids $w_i$ qui dépendent du $i^{ème}$ échantillon, le poids $w_i$ étant très faible lorsque la $i^{ème}$ donnée  est aberrante et important dans le cas contraire. De cette façon, le problème est réduit à l'optimisation suivante :<br>\n",
    "<h3><center> $\\min_{f}\\, \\frac{1}{n} \\sum_{i=1}^n w_i\\ell(f(X_i),y_i)$</center></h3> <br>\n",
    "\n",
    "Ref:[Guillaume Lecué, Matthieu Lerasle and Timothée Mathieu. “Robust classification via MOM minimization”, Machine Learning Journal (2020)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L'approche est mise en œuvre sous la forme d'un méta-algorithme qui prend comme entrée un estimateur de base<br>\n",
    "À chaque étape, l'algorithme estime les poids d'échantillon qui sont censés être petits pour les valeurs aberrantes et grands pour les valeurs incrustées, puis nous effectuons une étape d'optimisation en utilisant l'algorithme d'optimisation de l'estimateur de base.<br>\n",
    "Cet algorithme prend en charge deux schémas de pondération : Les poids de type Huber et les poids de la MoM. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Median-of-means weights:</b> le paramètre \"k\" est un entier non négatif, lorsque k=0 l'estimateur est exactement le même que l'estimateur de base et lorsque k=taille_de_l'échantillon/2 l'estimateur est très robuste mais moins efficace sur les inliers. Une bonne heuristique consiste à choisir k comme estimation du nombre d'observations aberrantes. En pratique, si k=Nul, il est estimé en utilisant le nombre de points éloignés de la médiane de plus de 1,45 fois l'intervalle interquartile."
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "TY  - JOUR\n",
    "AU  - Lecué, Guillaume\n",
    "AU  - Lerasle, Matthieu\n",
    "AU  - Mathieu, Timlothée\n",
    "PY  - 2020\n",
    "DA  - 2020/08/01\n",
    "TI  - Robust classification via MOM minimization\n",
    "JO  - Machine Learning\n",
    "SP  - 1635\n",
    "EP  - 1665\n",
    "VL  - 109\n",
    "IS  - 8\n",
    "AB  - We present an extension of Chervonenkis and Vapnik’s classical empirical risk minimization (ERM) where \n",
    "the empirical risk is replaced by a median-of-means (MOM) estimator of the risk.\n",
    "The resulting new estimators are called MOM minimizers. While ERM is sensitive to corruption of the dataset for\n",
    "many classical loss functions used in classification, we show that MOM minimizers behave well in theory, in the sense that \n",
    "it achieves Vapnik’s (slow) rates of convergence under weak assumptions: the functions in the hypothesis class are \n",
    "only required to have a finite second moment and some outliers may also have corrupted the dataset.\n",
    "We propose algorithms, inspired by MOM minimizers, which may be interpreted as MOM version of block stochastic gradient\n",
    "descent (BSGD). The key point of these algorithms is that the block of data onto which a descent step is performed is chosen\n",
    "according to its “ centrality” among the other blocks. This choice of “ descent block” makes these algorithms robust to \n",
    "outliers; also, this is the only extra step added to classical BSGD algorithms. As a consequence, classical BSGD algorithms\n",
    "can be easily turn into robust MOM versions. Moreover, MOM algorithms perform a smart subsampling which may help to reduce \n",
    "substantially time computations and memory resources when applied to non linear algorithms. These empirical performances are\n",
    "illustrated on both simulated and real datasets.\n",
    "SN  - 1573-0565\n",
    "UR  - https://doi.org/10.1007/s10994-019-05863-6\n",
    "DO  - 10.1007/s10994-019-05863-6\n",
    "ID  - Lecué2020\n",
    "ER  - "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Robust\n",
    "#Median of Means\n",
    "#Implémentation inspirée par:\n",
    "#[Stanislav Minsker and Timothée Mathieu. “Excess risk bounds in robust empirical risk minimization” arXiv preprint (2019). arXiv:1910.07485.]\n",
    "#[https://github.com/lecueguillaume]\n",
    "# Modifié pour Projet PIST par KERMADI ZIAD\n",
    "\n",
    "def blockMOM(K,x):\n",
    "    '''Sample the indices of K blocks for data x using a random permutation\n",
    "    Parameters\n",
    "    ----------\n",
    "    K : int\n",
    "        number of blocks\n",
    "    x : array like, length = n_sample\n",
    "        sample whose size correspong to the size of the sample we want to do blocks for.\n",
    "    Returns \n",
    "    -------\n",
    "    list of size K containing the lists of the indices of the blocks, the size of the lists are contained in [n_sample/K,2n_sample/K]\n",
    "    '''\n",
    "    b=int(np.floor(len(x)/K))\n",
    "    nb=K-(len(x)-b*K)\n",
    "    nbpu=len(x)-b*K\n",
    "    perm=np.random.permutation(len(x))\n",
    "    blocks=[[(b+1)*g+f for f in range(b+1) ] for g in range(nbpu)]\n",
    "    blocks+=[[nbpu*(b+1)+b*g+f for f in range(b)] for g in range(nb)]\n",
    "    return [perm[b] for  b in blocks]\n",
    "\n",
    "def MOM(x,blocks):\n",
    "    '''Compute the median of means of x using the blocks \n",
    "    Parameters\n",
    "    ----------\n",
    "    x : array like, length = n_sample\n",
    "        sample from which we want an estimator of the mean\n",
    "    blocks : list of list, provided by the function blockMOM.\n",
    "    Return\n",
    "    ------\n",
    "    The median of means of x using the block blocks, a float.\n",
    "    '''\n",
    "    means_blocks=[np.mean([ x[f] for f in ind]) for ind in blocks]\n",
    "    indice=np.argsort(means_blocks)[int(np.ceil(len(means_blocks)/2))]\n",
    "    return means_blocks[indice],indice\n",
    "\n",
    "class logregMOM_binary(BaseEstimator):\n",
    "    '''Class of the binary classification for the logistic regression MOM.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    w0 : array-like, length = n_features + 1, default ones(n_features + 1)\n",
    "        initial coefficients (including the intercept) of the classifier.\n",
    "    K : int, default 10\n",
    "        number of blocks for the computation of the MOM. A big value of K deals with more outliers but small values of K are better for the performance when there are no outliers.\n",
    "        \n",
    "    eta0 : float, default 1\n",
    "        step size parameter, the step size is defined as the i-th iteration by 1/(1+eta0*i).\n",
    "    beta : float, default 1\n",
    "        L2 regularization parameter.\n",
    "    epoch : int, default 200\n",
    "        number of iterations before the end of the algorithm.\n",
    "    agg : int, default 3\n",
    "        number of runs of the algorithm on which we aggregate. One might want to decrease this number if the complexity is a problem.\n",
    "    compter : boolean, default False\n",
    "        used for outlier detection, if compter=True, the number of time each point is used in the algorithm will be recorded in the attribute \"counts\".\n",
    "    progress : boolean, default False\n",
    "        display a progress bar to monitor the algorithm on each run (agg > 1 means several progress bar).\n",
    "    verbose : boolean, default True\n",
    "        display a message at the end of each run if agg > 1.\n",
    "    multi : {'ovr','ovo'} , default 'ovr'\n",
    "        method used to go from binary classification to multiclass classification. 'ovr' means \"one vs the rest\" and 'ovo' means \"one vs one\" .\n",
    "        \n",
    "    Attributes\n",
    "    ----------\n",
    "    \n",
    "    w0 : array like, length = n_features + 1\n",
    "        w0 is updated in the algorithm, provides with the final coefficients of the decision function.\n",
    "    counts : array like, length = n_sampled\n",
    "        the i-th element record the number of time the i-th element of the training dataset X has been used. Only if compter=True.\n",
    "    Methods\n",
    "    -------\n",
    "    fit(X,y) : fit the model\n",
    "        X : numpy matrix size = (n_samples,n_features)\n",
    "        y : array like, length = n_samples\n",
    "    predict(X) : predict the class of the points in X\n",
    "        X : numpy matrix size = (n_samples,n_features)\n",
    "        returns array-like, length = n_samples.\n",
    "    predict_proba(X) : predict the probability that each point belong to each class.\n",
    "        X : numpy matrox size = (n_samples,n_features)\n",
    "        returns matrix, size = (n_samples,n_class)\n",
    "     '''   \n",
    "    def __init__(self,w0=None,K=10,eta0=1,beta=1,epoch=200,agg=3,compter=False,progress=False,verbose=True,power=2/3):\n",
    "        args, _, _, values = inspect.getargvalues(inspect.currentframe())\n",
    "        values.pop(\"self\")\n",
    "        for arg, val in values.items():\n",
    "            setattr(self, arg, val)\n",
    "    def fit1(self,x,Y):\n",
    "        w=np.array(self.w0)\n",
    "        X=np.hstack([np.array(x),np.ones(len(x)).reshape(len(x),1)])\n",
    "\n",
    "        pas=lambda i : 1/(1+self.eta0*i)**self.power\n",
    "        if self.compter:\n",
    "            self.counts=np.zeros(len(X))\n",
    "        compteur=1\n",
    "        fincompteur=1\n",
    "        if self.progress:\n",
    "            Bar=progressbar(self.epoch)\n",
    "        for f in range(self.epoch):\n",
    "            if self.progress:\n",
    "                Bar.update(f)\n",
    "            losses=self.perte(X,Y,w)\n",
    "            blocks=blockMOM(self.K,X)\n",
    "\n",
    "            compteur+=1\n",
    "            risque,b=MOM(losses,blocks)\n",
    "            Xb=X[blocks[b]]\n",
    "            yb=Y[blocks[b]]\n",
    "            #IRLS avec regularisation L2\n",
    "            eta=self.sigmoid(Xb.dot(w.reshape([len(w),1]))).reshape(len(Xb))\n",
    "            D=np.diag(eta*(1-eta))\n",
    "            w=w*(1-pas(f))+pas(f)*np.linalg.inv(np.transpose(Xb).dot(D).dot(Xb)+self.beta*np.eye(len(X[0]))).dot(np.transpose(Xb).dot(yb-eta)-self.beta*w)\n",
    "            if self.compter:\n",
    "                self.counts[blocks[b]]+=1\n",
    "\n",
    "            \n",
    "        return w\n",
    "\n",
    "    def fit(self,x,Y):\n",
    "        if self.w0 is None:\n",
    "            self.w0=np.zeros(len(x[0])+1)\n",
    "        y=np.array(Y).copy()\n",
    "        self.values=np.sort(list(set(Y)))\n",
    "        yj=y.copy()\n",
    "        indmu=yj!=self.values[1]\n",
    "        indu=yj==self.values[1]\n",
    "        yj[indmu]=0\n",
    "        yj[indu]=1\n",
    "        w=np.zeros(len(self.w0))\n",
    "        for f in range(self.agg):\n",
    "            if self.agg !=1 and self.verbose:\n",
    "                print('Passage '+str(f))\n",
    "            w+=self.fit1(x,yj)\n",
    "        self.w=w/self.agg\n",
    "\n",
    "    def perte(self,X,y,w):\n",
    "        pred=X.dot(w.reshape([len(w),1]))\n",
    "        pred=pred.reshape(len(X))\n",
    "        return np.log(1+np.exp(-(2*y-1)*pred))\n",
    "\n",
    "    def predict(self,x):\n",
    "        X=x.copy\n",
    "        X=np.hstack([x,np.ones(len(x)).reshape(len(x),1)])\n",
    "\n",
    "        pred=(X.dot(self.w.reshape([len(self.w),1]))).reshape(len(X))\n",
    "        return np.array([self.values[int(p>0)] for p in pred])\n",
    "\n",
    "    def predict_proba(self,x):\n",
    "        X=x.copy\n",
    "        X=np.hstack([x,np.ones(len(x)).reshape(len(x),1)])\n",
    "        pred=self.sigmoid(X.dot(self.w.reshape([len(self.w)])))\n",
    "        return np.array([[1-p,p] for p in pred])\n",
    "\n",
    "    def sigmoid(self,x):\n",
    "        return 1/(1+np.exp(-x))\n",
    "\n",
    "    def score(self,x,y):\n",
    "        pred=self.predict(x)\n",
    "        return np.mean(pred==np.array(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>=> max_iter & eta0</b> sont des choix importants et je devrais mettre un cvgridsearch pour leur tuning mais on diffère cet exercice pour un autre temps ( Il est 5h du matin ^^ )<br>\n",
    "<b>=> Epoch</b> plus grande donne des résultats meilleurs mais dans plus de temps<br>\n",
    "<b>=> </b>Puisque cet algorithme a une loss function à base de calcul distanciel:Le <b>Scaling</b> des données est important et on fait donc un Scaling robuste des données avec RobustScaler\n",
    "<b>=> </b>Il est conseillé de choisir <b>K </b> tel que le nombre d'outliers si on les connait"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7752 ; Training Accuracy: 0.7769 ; Roc_Auc: 0.7531, ELA: 0.2323\n"
     ]
    }
   ],
   "source": [
    "#Application\n",
    "etapes = [('RobustScaling',  RobustScaler()), ('modele',logregMOM_binary(K=int(y_dirty.shape[0]*0.2)+17,epoch=500))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_logreg_mom_y_scaled = cross_validate(pipeline,X_dirty, LabelEncoder().fit_transform(y_dirty), scoring={'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_logreg_mom_y_scaled['test_accuracy']),\n",
    "                                                               np.mean(cv_logreg_mom_y_scaled['train_accuracy']),\n",
    "                                                               np.mean(cv_logreg_mom_y_scaled['test_roc_auc']),\n",
    "                                                               ELA(cv_logreg_mom_y_scaled)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Huber "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7787 ; Training Accuracy: 0.7729 ; Roc_Auc: 0.7516, ELA: 0.2286\n"
     ]
    }
   ],
   "source": [
    "#Application\n",
    "#TypeError: cleanlab requires zero-indexed labels (0,1,2,..,m-1), but in your case: np.unique(s) = [2 4] --> On utilise LabelEncoder\n",
    "\n",
    "etapes = [('RobustScaling',  RobustScaler()),\n",
    "          ('modele',SGDClassifier(loss=\"huber\",alpha=0.001,random_state=1,penalty='l1',class_weight='balanced',epsilon=0.3))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_logreg_huber_y_scaled = cross_validate(pipeline,X_dirty, y_dirty, scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_logreg_huber_y_scaled['test_accuracy']),\n",
    "                                                               np.mean(cv_logreg_huber_y_scaled['train_accuracy']),\n",
    "                                                               np.mean(cv_logreg_huber_y_scaled['test_roc_auc']),\n",
    "                                                               ELA(cv_logreg_huber_y_scaled)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Confident Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Le CL s'appuie sur les principes développés dans la littérature traitant des étiquettes bruyantes :\n",
    "\n",
    "><b>Pruning</b> pour rechercher les erreurs d'étiquetage, par exemple en suivant l'exemple de Natarajan et al. (2013) ; van Rooyen et al. (2015) ; Patrini et al. (2017), en utilisant un élagage doux via une repondération des pertes, pour éviter les pièges de convergence d'un réétiquetage itératif<br>\n",
    "><b>Counting</b> pour s'entraîner sur des données propres, en évitant la propagation d'erreurs dans les poids de modèle appris par la repondération de la perte (Natarajan et al., 2017) avec des probabilités prédites imparfaites, en généralisant les travaux fondateurs Forman (2005, 2008) ; Lipton et al. (2018).<br>\n",
    "><b>Ranking</b> pour classer les exemples à utiliser pendant la formation, pour permettre l'apprentissage avec des probabilités non normalisées ou des distances aux limites de décision du MVC, en s'appuyant sur les résultats de robustesse bien connus de PageRank (Page et al., 1997) et sur les idées d'apprentissage de programmes dans MentorNet (Jiang et al.,2018)."
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "@misc{northcutt2019confidentlearning,\n",
    "   title={Confident Learning: Estimating Uncertainty in Dataset Labels},\n",
    "   author={Curtis G. Northcutt and Lu Jiang and Isaac L. Chuang},\n",
    "   year={2019},\n",
    "   eprint={1911.00068},\n",
    "   archivePrefix={arXiv},\n",
    "   primaryClass={stat.ML}\n",
    "}\n",
    "\n",
    "@inproceedings{northcutt2017rankpruning,\n",
    " author={Northcutt, Curtis G. and Wu, Tailin and Chuang, Isaac L.},\n",
    " title={Learning with Confident Examples: Rank Pruning for Robust Classification with Noisy Labels},\n",
    " booktitle = {Proceedings of the Thirty-Third Conference on Uncertainty in Artificial Intelligence},\n",
    " series = {UAI'17},\n",
    " year = {2017},\n",
    " location = {Sydney, Australia},\n",
    " numpages = {10},\n",
    " url = {http://auai.org/uai2017/proceedings/papers/35.pdf},\n",
    " publisher = {AUAI Press},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application avec Regression logistique comme estimateur de base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7858 ; Roc_Auc: 0.7633, ELA: 0.1881\n"
     ]
    }
   ],
   "source": [
    "#Application avec Pruning par ratio de bruit -->  works by removing examples with\n",
    "        #*high probability* of being mislabeled for every non-diagonal in the `prune_counts_matrix`\n",
    "    \n",
    "conf = LearningWithNoisyLabels(clf=LogisticRegression(C=0.1),cv_n_folds=10,seed=1).fit(X_dirty,LabelEncoder().fit_transform(y_dirty))\n",
    "conf_test_accu=conf.score(X_dirty,LabelEncoder().fit_transform(y_dirty)) # Score Accuracy du Test\n",
    "conf_roc_auc=roc_auc_score(LabelEncoder().fit_transform(y_dirty),conf.predict(X_dirty)) # Score roc_auc\n",
    "conf_ela=1-conf_test_accu/np.mean(cv_clean_y['test_accuracy']) # Score ELA\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (conf_test_accu,\n",
    "                                                               conf_roc_auc,\n",
    "                                                               conf_ela))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7858 ; Roc_Auc: 0.7633, ELA: 0.1881\n"
     ]
    }
   ],
   "source": [
    "#Application avec Pruning par classe --> works by removing the examples with *smallest\n",
    "       # probability* of belonging to their given class label for every class\n",
    "conf_c = LearningWithNoisyLabels(clf=LogisticRegression(C=0.1),cv_n_folds=10,seed=1,prune_method='prune_by_class').fit(X_dirty,LabelEncoder().fit_transform(y_dirty))\n",
    "conf_c_test_accu=conf_c.score(X_dirty,LabelEncoder().fit_transform(y_dirty))\n",
    "conf_c_roc_auc=roc_auc_score(LabelEncoder().fit_transform(y_dirty),conf_c.predict(X_dirty))\n",
    "conf_c_ela=1-conf_c_test_accu/np.mean(cv_clean_y['test_accuracy'])\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (conf_c_test_accu,\n",
    "                                                               conf_c_roc_auc,\n",
    "                                                               conf_c_ela))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# application avec RF comme estimateur de base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7994 ; Roc_Auc: 0.7826, ELA: 0.1741\n"
     ]
    }
   ],
   "source": [
    "conf_rf = LearningWithNoisyLabels(clf=RandomForestClassifier(),cv_n_folds=10,seed=1).fit(X_dirty,LabelEncoder().fit_transform(y_dirty))\n",
    "conf_rf_test_accu=conf_rf.score(X_dirty,LabelEncoder().fit_transform(y_dirty))\n",
    "conf_rf_roc_auc=roc_auc_score(LabelEncoder().fit_transform(y_dirty),conf_rf.predict(X_dirty))\n",
    "conf_rf_ela=1-conf_rf_test_accu/np.mean(cv_clean_y['test_accuracy'])\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (conf_rf_test_accu,\n",
    "                                                               conf_rf_roc_auc,\n",
    "                                                               conf_rf_ela))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RobustLR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modifies the logistic loss using class dependent (estimated) noise rates for robustness. This implementation is for binary classification tasks only."
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "@inproceedings{NIPS2013_3871bd64,\n",
    " author = {Natarajan, Nagarajan and Dhillon, Inderjit S and Ravikumar, Pradeep K and Tewari, Ambuj},\n",
    " booktitle = {Advances in Neural Information Processing Systems},\n",
    " editor = {C. J. C. Burges and L. Bottou and M. Welling and Z. Ghahramani and K. Q. Weinberger},\n",
    " pages = {1196--1204},\n",
    " publisher = {Curran Associates, Inc.},\n",
    " title = {Learning with Noisy Labels},\n",
    " url = {https://proceedings.neurips.cc/paper/2013/file/3871bd64012152bfb53fdf04b401193f-Paper.pdf},\n",
    " volume = {26},\n",
    " year = {2013}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7777 ; Training Accuracy: 0.7833 ; Roc_Auc: 0.7608, ELA: 0.2296\n"
     ]
    }
   ],
   "source": [
    "#Application\n",
    "#On utilise LabelEncoder\n",
    "\n",
    "etapes = [('StandardScaling',  StandardScaler()),\n",
    "    ('modele',RobustLR(NP=0,PN=0.2 ,random_state=1 ))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_logreg_rob_y_scaled = cross_validate(pipeline,X_dirty,LabelEncoder().fit_transform(y_dirty), scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_logreg_rob_y_scaled['test_accuracy']),\n",
    "                                                               np.mean(cv_logreg_rob_y_scaled['train_accuracy']),\n",
    "                                                               np.mean(cv_logreg_rob_y_scaled['test_roc_auc']),\n",
    "                                                               ELA(cv_logreg_rob_y_scaled)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7863 ; Training Accuracy: 0.7979 ; Roc_Auc: 0.7689, ELA: 0.2208\n"
     ]
    }
   ],
   "source": [
    "#Application\n",
    "etapes = [ ('modele',RandomForestClassifier(n_estimators=301,\n",
    "                                             random_state=1,\n",
    "                                   class_weight='balanced'\n",
    "                                   ,max_features=3\n",
    "                                   ,max_depth=5\n",
    "                                   ,min_samples_leaf=2))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_r = cross_validate(pipeline,X_dirty, LabelEncoder().fit_transform(y_dirty), scoring={'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_r['test_accuracy']),\n",
    "                                                               np.mean(cv_r['train_accuracy']),\n",
    "                                                               np.mean(cv_r['test_roc_auc']),\n",
    "                                                               ELA(cv_r)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "La RF hérite son insensibilité aux valeurs aberrantes du cloisonnement récursif et de l'ajustement des modèles localement(les leafs)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RobustRF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uses a random forest to to compute pairwise similarity/distance, and then a simple K Nearest Neighbor that works on that similarity matrix. For a pair of samples, the similarity value is proportional to how frequently they belong to the same leaf"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "@InProceedings{pmlr-v70-li17e, \n",
    "title = {Forest-type Regression with General Losses and Robust Forest}, \n",
    "author = {Alexander Hanbo Li and Andrew Martin}, \n",
    "booktitle = {Proceedings of the 34th International Conference on Machine Learning}, \n",
    "pages = {2091--2100},\n",
    "year = {2017}, \n",
    "editor = {Doina Precup and Yee Whye Teh}, \n",
    "volume = {70}, \n",
    "series = {Proceedings of Machine Learning Research}, \n",
    "address = {International Convention Centre, Sydney, Australia}, \n",
    "month = {06--11 Aug}, \n",
    "publisher = {PMLR}, \n",
    "pdf = {http://proceedings.mlr.press/v70/li17e/li17e.pdf},\n",
    "url = {http://proceedings.mlr.press/v70/li17e.html}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7853 ; Training Accuracy: 0.7902 ; Roc_Auc: 0.7686, ELA: 0.2219\n"
     ]
    }
   ],
   "source": [
    "#Application\n",
    "#On utilise LabelEncoder\n",
    "#K=51\n",
    "etapes = [\n",
    "    ('modele',RobustForest(n_estimators=301,max_leaf_nodes=5,random_state=1,K=51,method='weighted'))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_rf_rob_y_scaled = cross_validate(pipeline,X_dirty,LabelEncoder().fit_transform(y_dirty), scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_rf_rob_y_scaled['test_accuracy']),\n",
    "                                                               np.mean(cv_rf_rob_y_scaled['train_accuracy']),\n",
    "                                                               np.mean(cv_rf_rob_y_scaled['test_roc_auc']),\n",
    "                                                               ELA(cv_rf_rob_y_scaled)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extremely Randomized Trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class implements a meta estimator that fits a number of randomized decision trees (a.k.a. extra-trees) on various sub-samples of the dataset and uses averaging to improve the predictive accuracy and control over-fitting"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "@article{Geurts2006ExtremelyRT,\n",
    "  title={Extremely randomized trees},\n",
    "  author={P. Geurts and D. Ernst and L. Wehenkel},\n",
    "  journal={Machine Learning},\n",
    "  year={2006},\n",
    "  volume={63},\n",
    "  pages={3-42}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7883 ; Training Accuracy: 0.7949 ; Roc_Auc: 0.7713, ELA: 0.2188\n"
     ]
    }
   ],
   "source": [
    "#J'ai manipulé les params des arbres pour réduire l'overfitting\n",
    "etapes = [\n",
    "    ('modele',ExtraTreesClassifier(n_estimators=301,random_state=1,\n",
    "                                   class_weight='balanced'\n",
    "                                   ,max_features=3\n",
    "                                   ,max_depth=5\n",
    "                                   ,min_samples_leaf=2))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_etc = cross_validate(pipeline,X_dirty,LabelEncoder().fit_transform(y_dirty), scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_etc['test_accuracy']),\n",
    "                                                               np.mean(cv_etc['train_accuracy']),\n",
    "                                                               np.mean(cv_etc['test_roc_auc']),\n",
    "                                                               ELA(cv_etc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RANSAC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L'algorithme divise les données complètes de l'échantillon d'entrée en un ensemble d'intrants, qui peuvent être sujets au bruit, et d'aberrations, qui sont par exemple causées par des mesures erronées ou des hypothèses non valables sur les données. Le modèle qui en résulte est alors estimé uniquement à partir des valeurs intrinsèques déterminées."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7672 ; Training Accuracy: 0.7765 ; Roc_Auc: 0.7377, ELA: 0.2405\n"
     ]
    }
   ],
   "source": [
    "# Application avec Regression Logistique comme estimateur de base\n",
    "# les residual_threshold sont par defaut la mean deviation du set des labels qui est 0 ici, on le définit don manuellement\n",
    "etapes = [('StandardScaling',  StandardScaler()),\n",
    "    ('modele',RANSACRegressor(base_estimator=LogisticRegression(),random_state=1,residual_threshold=3))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_ransac = cross_validate(pipeline,X_dirty,y_dirty, scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_ransac['test_accuracy']),\n",
    "                                                               np.mean(cv_ransac['train_accuracy']),\n",
    "                                                               np.mean(cv_ransac['test_roc_auc']),\n",
    "                                                               ELA(cv_ransac)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RANSAC n'a pas d'effet, ce qui est logique puisque le RANSAC n'essaie pas de tenir compte des valeurs aberrantes, il est conçu pour les cas où les points de données n'ont vraiment pas leur place, et non pas seulement pour ceux qui sont distribués de manière non normale. Le bruit ici n'est pas suffisament marqué."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7777 ; Training Accuracy: 0.9326 ; Roc_Auc: 0.7560, ELA: 0.2297\n"
     ]
    }
   ],
   "source": [
    "etapes = [('StandardScaling',  StandardScaler()),\n",
    "    ('modele',KNeighborsClassifier(weights='distance',n_neighbors=17))]\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_knn = cross_validate(pipeline,X_dirty,y_dirty, scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_knn['test_accuracy']),\n",
    "                                                               np.mean(cv_knn['train_accuracy']),\n",
    "                                                               np.mean(cv_knn['test_roc_auc']),\n",
    "                                                               ELA(cv_knn)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.7813 ; Training Accuracy: 0.7813 ; Roc_Auc: 0.7627, ELA: 0.2260\n"
     ]
    }
   ],
   "source": [
    "etapes = [('StandardScaling',  StandardScaler()),\n",
    "    ('modele',SVC(C=0.1,random_state=1))] # C petit pour pénalization plus forte sur les outliers\n",
    "pipeline = Pipeline(steps=etapes)\n",
    "cv_svc = cross_validate(pipeline,X_dirty,y_dirty, scoring={\n",
    "                                                       'accuracy':make_scorer(accuracy_score),\n",
    "                                                       'specificity': make_scorer(recall_score,pos_label=4,average='weighted'),\n",
    "                                                       'roc_auc':make_scorer(roc_auc_score) }, \n",
    "                             cv=cv, n_jobs=-1,return_estimator=True, return_train_score=True)\n",
    "\n",
    "print('Testing Accuracy: %.4f ; Training Accuracy: %.4f ; Roc_Auc: %.4f, ELA: %.4f' % (np.mean(cv_svc['test_accuracy']),\n",
    "                                                               np.mean(cv_svc['train_accuracy']),\n",
    "                                                               np.mean(cv_svc['test_roc_auc']),\n",
    "                                                               ELA(cv_svc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Theilsen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pour données numériques"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Détection d'Outliers - Apprentissage non supervisé"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnQAAADHCAYAAACQqM49AAAgAElEQVR4Ae2dB9QlRZXHOR4RFxhQBEVQggxhlUVyUpAsKCsCYgJnRYY0gAFExF1E0XEAERwkZ2aIM6QFMSAMCiwwBAWVWUVYsgTJSB6oPb967/a7Xa/7he+Fee/7/nVOv+7Xlf9VXfXvW7duzxfkhIAQEAJCQAgIASEgBIYagfmGuvQqvBAQAkJACAgBISAEhEAQoVMnEAJCQAgIASEgBITAkCMgQjfkDajiCwEhIASEgBAQAkJAhE59QAgIASEgBISAEBACQ46ACN2QN6CKLwSEgBAQAkJACAgBETr1ASEgBISAEBACQkAIDDkCInRD3oAqvhAQAkJACAgBISAEROjUB4SAEBACQkAICAEhMOQIiNANeQOq+EJACAgBISAEhIAQEKFTHxACQkAICAEhIASEwJAjIEI35A2o4gsBISAEhIAQEAJCQIROfUAICAEhIASEgBAQAkOOgAjdkDegii8EhIAQEAJCQAgIARE69QEhIASEgBAQAkJACAw5AiJ0Q96AKr4QEAJCQAgIASEgBETo1AeEgBAQAkJACAgBITDkCIjQDXkDqvhCQAgIASEgBISAEBChUx8QAkJACAgBISAEhMCQIyBCN+QNqOILASEgBISAEBACQkCETn1ACAgBISAEhIAQEAJDjoAI3ZA3oIovBIRAcwSeeuqp8Oijj4ZHHnkk3HfffeGee+4Jb775Zhbx+eefD48//nh4+OGHw7333hv++te/hjfeeCPz14UQ6CcCc+fODd04rMzPPvtsSA+eh7///e/hwQcfDE888UTgGXn66adj/ydvueFDQIRu+NpMJU4QeOGFF8Jtt90Wpk+fHv7v//4v8W3/L5M9af3lL38JL7/8cvsJKMZAIQBZW2211cJaa60V1l577TDffPOFf/u3f4uTGQX95z//GT7+8Y9Hf8IstNBCMQwTnZwQmBcI/Md//Ef40pe+1NGx0047hYceeii8+uqrges111wzd3z4wx8Odqy++uphjTXWiP4bbrhh2HvvvcPFF18cX3DmRf2V58gQEKEbGW6KNUAIXHjhheH9739/nIS33377cP/994+4dMTdYYcdYlof+tCHwh/+8IcRp6WIg4HASy+9FL74xS9G0gaZ44DYIZ3AMeHtvPPOgb7DZGZhROgGo/3GWilef/31MGHChKwfLrDAAmG55ZYL6623XiRdCy+8cOZHX4WIQcx4GVl66aVzfn/7299i//7CF74QSR1hrX9z5kWH8Q4CybHddtuF8ePHB/JkTP3a174WbrnlligtHGvtMIz1FaEbxlZTmXMI+MGPQep///d/c/7t/Pn973+fG/B+8pOftBNdYQcUgVdeeSUuJ0Hs6COe0FFk/FmSMn/CiNANaGOO8mK9+OKL4ZOf/GTsp1/5ylfCTTfdFJdEn3zyyfiyOnny5GyMgpDx0smqwmOPPRYlarvvvnvmP2vWrIgWLy2sZPDCijSO/s1x7LHHBvLjIAzPAMuvM2bMyF5uPvjBD4Zf/OIXoxz10VE9EbrR0Y5juhbXX399fINlqYyls3/84x8jxoMliq233jrwFrzSSivFgXLEiSniwCGAFKKI0FlBv/zlL2eTnQidoaJzPxGAdK2zzjphs802y6TIPv8TTjgh66Msoz733HPeO14fdthhMcwpp5yS80O9wEvpTjrppJy//zNt2rQsHyTX3VBn8enruvsIiNB1H1Ol2GcEUF5Hif2uu+4KzzzzTMe58yaM/twDDzzQcVpKYLAQMGluKqGzUorQGRI6zysE2JzzgQ98IOy5556FRTjuuOMyolVG6JDq8VJ6+OGH59Joh9D96U9/yvLhJejss8/OpaU/g4eACN3gtYlKJASEQI8QEKHrEbBKtmsIsEqwyCKL1JExy6AVCR0vt5DCb3zjGxYtnlNCd/zxx+f8/R8k1LY0y3mfffbx3roeQARE6AawUXpVJPQk2JbOkiQK4aYUTn5sWcdkA2Yd0KMocgwG6FcQhkGHN0kc6RCHc5mpB/Q7brzxxoCSLtI0n3dRXtyjnIj50Yn785//HO6+++66eCgQs+SAVM3MUhSlh+SO+H/84x/D7bffHmbPnh3LjIkKdEdwmLEAH9IyjMyvKE12wFK+OXPmxF22pMmOyiKH/grYgTHYgbc58gNTBtAy7C1sv85gQV3ADSwoH3pmmDOgvGDJuczRN2gzsEa/B1ybOfrG//zP/8T+AaZgWxaPMtF2TFwc9Cnav5lj52CjJVcjfITRkmszNOXfCwQY7+h/V1xxRWHyrRA6xld2cn/uc5/LpZESukZLrimhO+KII3KmfnIJ689AICBCNxDN0PtCQOa22GKLqD/BbiiOVVddNSq/HnDAAWHLLbcM733ve8MSSywRNt9884AOhicdDAQf/ehHY3x0MIiPsuw555wTPvKRj8QBiPS/9a1vhddeey2rEITruuuuizus3ve+98U3zxVXXDHurGLpgKXN1EEmmKSZXDfaaKOw+OKLZ2+KW221Vfj5z38ezYmQNrsTKY8d1Cmd2Fk6/dSnPhUWXXTRmA47uJZZZpnw7//+71H37tRTT41FuPnmm7PdYizJvfOd7wxHH310Wrz4H7Lx7W9/O2JqO8vQuaM8d955Zw4DSInHjvp/5jOfiSSEJZF111035sVOts9+9rMRr8JM+3gTssSEwJIObb3kkkuG73//+3EnHG/+9JNdd921blkaIoeiNX0D/N71rnfFDQif/vSnw3//939HUphWA3yuvPLKsOOOO4aVV145a2smNSYk7MZ5R9/YZJNNcuHoW+hPfve7343K4T68v25G6EzHToTOo6brfiLAruxjjjkmvgAW5dsKoSONn/70p+H000/PJdEOoePliueAg+f98ssvz6WlP4OHgAjd4LVJT0qEtIUJGtJmDylnJkIUcL/61a+GqVOnhokTJ2b+6HCYxAkpWBr/X//1XwOmPbBZhB/pQeqQQuGQ5vz4xz/O0oMQQJo4Wzn222+/nKQHqRdEj63zpMeOLcySsFsLAsW9FVZYIdx6662RHLArEbLBfQ6ImCd07FrddNNNox87VtEtQekYogapI44RujPOOCNAthZccMEsPQbP1P32t78NlJu4kGHICFv7p0yZEkkyZUcZGYkWDmkSRNPqTDxMBYAVAyXLIgy+H/vYx2Kaq6yySsQpzbef/yHalJnyUV4OsH7Pe94Tl17s3qRJkzJbfUhhWZbBjxcElKp/9atfBUg49+hr6a5hyBwvAfiT9qGHHhrjgJ8pb4OLSeog54ccckgMz07Aa665JvYNiNw73vGOeJ9+gjS0yBmhg6hCWlNn/pQHKaScEBg0BFohdGVlbpXQ8bzxEsxzwLHvvvtGKX1Zuro/GAiI0A1GO/SlFCyRIe34xCc+kT2oSE785Mebnd/2zkRpxnWJzyRo8ZGebbzxxnFpinR48CFbNlEy2c4///zxfjqRX3rppfH+29/+9tzSAnFtEIFcGili6Q9bS+Z38sknR8wYeFhesKWylNARjjjsXE2dkVcjdEj8kEpCJiyflNBBCKk3/nvssUfdEjM7bsEA/6uuuirLElJ3xx13BMgafpgOIBz3zGEawEjJ9773vXlq+wnc6Rc/+9nPMixoqx/84AdRqmr4YLCUSYLwBx10UAy71FJLxSVtqxfL1uBPHGxpQepxLENDiC0t8jKH5Nfuc7Y+Sv/19z3poq/ihzI4UsIiZ4StbFOE+ZOOllyLENS9eY1ANwkdL1BIvFGHYbw+77zz4ou9vaDzcssqAioNcoOPgAjd4LdR10vol5XYyZQ6Hl6TXpk0zMKgI2fk6a1vfWuUTuGHRIfJ8Nprr406aUzWthSL1ItlSO/QFVt++eXjBLz//vtnNpBYBmUyRfrnBxHyPeuss6I0jnTTZTibiFNCx9Im6SGlQ3/Ou1/+8pfRz8ih+ZkOC/E8oYPYGtlFadmXz+ISxvAlT19vyKdJnUg7Xc5FsonkCD8MgUKURuqQUmJLCh22Tj7jg14b5eEAW0g9y/cM/gcffHBG3sHWJJCQ+1T38Ic//GGWju2Wu+SSSzJpKITPx4HAbbPNNjEOO0+N2IMJtrkozy677BJJoWGEnp9JFIlj0mXz52x9t4zQmT/pi9B55HQ9KAh0k9DZs1105vk799xzC82iDAoWKkceARG6PB5j4p8RjkaTlp/YWEo0B7FiIiXuYostVqdDZeFQhLelUHSbkEQhffOH6UExcEAIkbagR0baLLNBHLzjPySqiEhZeVNCh96WDVaQJcgjBI6lWEghGxS8zh/5ed0RT+hYqkVPjvSwol5EGIiPbpnlib6fOcKbUU90y8DIO6R46KsRl3Bp/X3YRtfkY8T4LW95S50eTaO4qZ9XjEYaW+bOPPPMrM5Yl/ftzLVvh//8z/+MyZitLOpLnNTxUkD+KQ6QOtoCfT0INGfyuPrqq6Ple9KjPxA/ddb3ywidzJakiOn/oCHQTUL3X//1X3FTF1I6XmQ5uOYFrci+3aBhofLkERChy+MxJv4Z+WHiK5NCfOc738km6K9//euZlMcTOsiXLYWlwCHBIf1WDnS1kK74OCzrtuPKJHQsx9oknpYFQokEKyUMZRI69PgsDZZ/y3aksrxs4bwdKE/oWHr1m06oK9KvbhA66jxu3LisDGzU8B+ibwdXdNasLmVtggTQ9NosbKMz5UEaR7+ycP6loZXyQYYvuOCCSAQpl+lcWnpIZlEfSJ31/TJCZ/6kU/ZspGnqvxDoJwLdJHToK8uNHgRE6EZPW7ZcE09wyiYt0z1jYiO8LXlBDIw8sZORJbki5yU2KK9jwgIpXNHB2yDp+oEKKWA7zibiVEJHGpgTocwsk9qEb2d20PplUcKzwcH8vYTOEzU+uZNK9qy8Rx11VBafcplDD9AkdEgLkch5xzKpLcl2IqFDgrXttttmZTAdQZ9Xq9dITQ0LdqEWOXCwzRCEZZNDUTvbPZaWIdFeGpbuxivKh3u8ULApZYMNNojlYsct/ZNNJbwQGL7sKk6JOvGt75cROvOnHmXPRlnZdF8I9AMBP04yjrQjSWNssDGGPi5C148W618eInT9w3pgcjLy02jSQj8Kfw4U4U0Pi40DFh9pUpmE7te//nUWf/311882VjQC4Xe/+10Wh12SlmejOOZnJDMldJAmllVZfmMpgR2pKAKbfhb1g4Aw0Jkrk9CxycEwYVAsc1666SV05GGEAwlf+iUKL6GDMPJ/pI4dp+eff34AUyPjI0mLpUyrM2SnyDGhUE8Lx4aOZg4SyO5ei5PqMZbF92Qb0kY/M907pHa2ccYkqBBADnNG2MoInSeZInSGms6DhEA3CV26WW2Q6qmytI+ACF37mA19DCNkTKZFk5ZfViXMRRddlKuznxTRZSpykCg2NhCfXY/oORU5llqRoEHemJBtFyimVNIlSYsPWUnLXUbokPSxYyuVhrEchy4X5UPHjzTNlRE66mR20iCzRRIg0sCWHOlypDp09nbMm3VaP0/oOpHQWT26caZ9rS7oBpa53/zmN1m4sk8WERdSbGZtWGa1tJGwFTnaCSkwZBh8TO+SeLa5wuKl+GH/EOnwiSeeaEEkocuQ0MWwItBNQicJ3bD2guJyi9AV4zKq73pCx+aA1LE0tuyyy8bJlo8yI6Uxh4TOpBiQGvS1ihyk0MxYMPli4y51hEGfiqVbiB0Eafvtt88meW/GwuIyaWPXjF2gJpnBz+qUSujsPtKq1Jl+GBsIPKG74YYbsjL4JVekSnvttVfmV2Rok2VVJEfUmbPfWQuBM0KHBCndqesJCf78n9fOmwmh3csc7YcRYKu3twVoccACySOK2Dikbab7BoH3bWBxsA1ImhdffHEk8SbhpO+l+NGXIcqEJxz2Drn2Ugj/MlLUd/2GlvSlwcqksxCYlwiI0M1L9Ac7bxG6wW6fnpTOSA6THSYmvA4GpMN2seLP7kRzbAJAUmL225iE2VSA5KpogwATIl9BIB3IDTbavDvttNOinTrO5rCHZF+GwKwJpkXMoWfH0hz+6FHhWPKFTHz+85+P+TDR33bbbZlxYZvAqbNJhiw9k9Ah3WNZkh2T1MUvrUIKSN+WZKkTJJc6YY/Oky7S8EvV/tM94AO2EE7icsY4si1ZkzdSRCMkEDqItflbmft1pjyUGdJDeTnAifJwFO3wheDydQjCQtQ9qQMnDBOz1G0kF0JvhI049EWPJ23BbmeW7CkHBM527xLe6wbyMkAbo1OHHzhC+rnmZYK60D677bZbvEc/oa9RF4i6+VvfJh7EHn+wkBMC8woB+iaScswe8YUaVGDonxy8uKCfzPPB+EHYIseLJs8PYRlbLL6ZjMKfQ264ERgRoWN5rNNjuGEb3tJDiozk8FBjWgRignV9lqaM7EGmMDJpy4oMJkjSMNpqgwFnSB2DCgZmkd6ljuVU+1IAduvIm28CbrfddjEdzFWky7EQP4tjSu9IWbBczsYG4jDJQkT5UgX58xUCKxdSMMoKobD6sPzLJ80wZDt58uRIWvkyATbybGMHX7VgsKNOlhaElCVbL+XBbMo3v/nNGAZ/vhrB8qFZVkcKBbkxCSKEkHJimNfS5QypoJyQU5ZyjcxZGP6b9DLFtdf/Iep+4LcyQUTBo8hwL8ujSA/MviDL5wceeGBApxCcMfkCSfIO3UZ2q1oc2hIpKFI88EGC5+Pw1RAzbYMRZjZ//OhHP4rGrlnaR3/Pymr4IbkDR0zNmB9n6sJ9JIW0j33CzcLQD7gP8ZYTAvMKAV6EGFPoz4xtCy20UK4f85xwWH/3L1KUmXGIXd88z7ZCYH2cM/Hw82PcvKqr8u0MgREROqQkvIEzWbZ7EO/II4/srNQDFps3fyQ36OxAgtg1yRsT5GkQnZEcHmZIDArtGMHFfAibEVg6Y5IzQkIdIHTcR0rDQRp2zRmpXhGhIy5Lm+yCRAKDpI48+FwY0hkkIEWON1LyQAeKAQdjwkhnMC5s0jIIHZI564OQRSsTBJOBDSkNAyJGbCkj6UBIMHUBMUQKZu7444+P4UmPtAhPetQbP++Q9iEhIg1IBZM/ZSU/CJ8RYeJA6NghamUjbSsznwCD0HE2f5Y2CcNBPOL322E3EAytTJTXrtERLNvEgIQNO37EBWMmENqNjSh8ZaOoj9DP6G+EQ0eO9qGdyI+dq97xTJEO/ZYyEZZ+S1g2vdBPkbxCDLFzd9lll0VpYtpPrI3BF8JGH7F24Ux6hKEe/mseviy6FgL9QAA1APon45E/6J/2n/7KAXErInQ2llk44to1Z55pr17Sj3opj+4j0DahY0Clc/DGkLJ9Jl7eFGD7HAyqnG0Zxt4KRpMiJsQNMxW8yVv9+NwV+GCKARH3oDkmLCsrkhgmWSYtyoqZCk9GfNkb7Tpl2aqZg4CBFxK5sqWBNA2IMpslWH4rWhKg7EXE2cpKHMsLIgiBZNmC5QlPWMmXdIoIh/mlZeM/S4/ofjGIFpXP4rC8WOQMtyJ/7pWVpyitbt4DiyJcyaORn5WBskPurE8ZCTf/ojPtwXIsS+ZsTCnrh8QlLGlDnulTvi3BDGLHi5Y57lmfsHucwd/a3ftb3c3fx9G1EOgnAjxLHNYn07y5jz/91z8HPlxZXAuDf7MwFlbnwUWgbUJHVZiQmRSRSBkxsKURBnEmYQ4mOSQtvDnzho51fMLzHc/R4JhIILUspaHMzzIU9bRvfVJXdIC8FGhe15uH3hM62klOCAgBISAEhIAQGG4ERkTorMr+cz4spxUpSltY3iD4mDkkZ7TopKAwv+iii0adLP92A+H1pAlxdtEOPsOmn2fK6cuGNEROCAgBISAEhIAQGG4EOiJ06A+ZhK4VY6IoL6NUn5obGEYIkc4hlWMHHsrw3rQH9cH+2Lvf/e4Mn3SHZ7/rzLIjS90om6Ogbu3GBgE+nJ7uAO13+ZSfEBACQkAICAEhMHIERkzo0BfiY91GDDD14B2SICRVXgeInYHsZhsUaZUvb7vXLCOzy87qv+++++aSAJ/NNtss8z/mmGNy/v3+A6EzQ79WZn/GhIOcEBACQkAICAEhMJwIjJjQoUO32GKLRcKCaYlUFwt/CIO3FcUOVyRa3u7ZcMIWom4gu/iMFPl6UifqyI4j8+d6XjqWvDHPgXSUY86cOVHxnB25HK0orc/L8itvISAEhIAQEAJCoByBERO6GTNmZGQFY6HYk/KO7zRi4wxiZw7zDBAJL7Uzv2E8o0OIZA6zH6lpCaSQkFcjdNOnTx/GKqrMQkAICAEhIASEwBAgMGJCh4VpIysYVmVDBIZFMReAHSpMdyDBamTKoVv4sLOWJcVOD5aI23Xe1IGPi/0swweTJkjB5ISAEBACZQggRe90DLP4o2EVpAwn3RcCQqAYgREROpZX/Sd4ML7JzkkMIGKHzixZozPXa4ddsfXWWy+aD8GESNmBTbwiP+7b0a3lYGxXYUjXCB2W8lMJZq9xUfpCQAgMFwJI+jGUXDRO+XvNxjJsf6Y6vcOFhEorBITASBAYEaFjR6d9vJ1PH5khYQYd//mcRsuMvI3ySZ6f/vSnhQY/W60Mul9YvYZQNjqKwmCR3x9YzO6GRJEdrksssUQkdFtvvXU0ftpqfdhMgiHdWbNmlX5FodW0FE4ICIHhQYCVjaJxKh3XGKf8PT+G2TWfSJMTAkJgbCEwIkLHBgCTPtmniTB7gZgfMgLBW3DBBeO1hxMr1liAxwAvmwTYVMFnlcqWLX3cYbnGcr3tbsWocLsmWtDLw7Yd+GLbr0iyhx0/jDrrEAbqA4PbB4ZlzLJyqi8Nbl9S26ht0j5gz60/j4jQ8V05I3TYNfMOyRtvjywdoE/nHdIv4kH4eMvkmm9fjhZCxzdLWX6mXnybM93567Eouzbjy6Txtre9LX4HNQ0LocNfhzBQHxjMPsC3Y4fN8W1l9afB7E9qF7WL7wN8r7rItU3oIGnenhkSJe8gbXzTtczQMP5sYvjqV78aB4/RQuj4NujBBx8c68S3bpFEmkNy2aqSsh9U+YRYUTwI3dFHH61DGKgPDHAfsOd/WM6MPRpXNK6qDwxHHygaV9omdHw825jiuHHjws0331yXLsuuzb480C1Cxwe4jz/++HDiiSd2fPzsZz+LO3XrKtTkBsuihxxySMSFr2d4EyZs2lhllVWibkyTZKI39bnyyisDZUHiJycEhMDYQICvz3RjHCONG264QR9bHxvdRrUUAhkCbRO60047LSN0fClipAZpjdChJ9bJkis6assss0xWJiObIzmzFHz//fdn4LRyQf35piv5HXHEEXV48K1Ulk7POuusVpJTGCEgBMYoAtjuHMm4VRQHiwOjxd7nGO0OqrYQaBuBtgkdNudsAGEAGqkzQsfAw2aJkTrI4N133x0eeuihjg/SaYdcoi/43e9+N+IBLpA3DCnbgaTSyB472OSEgBAQAmUIYO6oG+MYaXRjt35ZOXVfCAiBwUSgJUKH0WD03v70pz+F7bbbLiN0v/71r2OtXn755cLdmI2qbITuwAMPHNo3ST52b+S20Zldq5A7OSEgBISAEBhbCDB//upXvwoTJ04ckUrP2EJLte0EgaaEjk9YYZqE5UjszJnRYAgM33Dl3gc/+MG4s7Wdghihw5jmMC4NoCOIblwjImd+q666agBHOSEgBAYfgWbjEVJ87EXKCYEyBFDFYZPJ5MmTI5F797vfHdZZZx0RujLAdL8rCDQldIjvsRXnDV76ayR2EJZ2dcSM0O21115DSehQYIbIcqy44oph+eWXj0aVeXDf+c53hoUXXji8/e1vD295y1uiaZbUhEtXWm+MJfLoo4/GjSLdmEyRKkOyOcuNXQQgZyeffHKYOnVq2HvvveOLKSaH7BvU9LVTTjkl2s7k6wuYZOIF9y9/+cvYBU01b4oAhA7TXdhb/clPfhJf/EXomsKmAB0i0JTQMaC9+OKLcUmV3Zzou9nZrjGtwe7MdtywEzrqCi6tHu1go7D1CLC8/5GPfCQsueSS4ZhjjulIQoKJmW233TYOsnzCDnUCubGJAGMYk65J0+1shA492SJ/9G3lhEAjBHhZRC+SjXb0q24QOr43fsIJJ8SvLDXKW37DhcBLL70UfvzjH+fMnY2kBk0J3UgSbSWOETreipstcbSSnsKMXgQgzRiwtsl2m222yZmGabfmmIWxtDhffvnl7Sah8KMEAQgbqxBXXHFF4Msu1i+M0PFCi2T4l7/8ZfjsZz+b+YvQjZIO0IdqdIvQ0SeRENNH+SKR3OhBAEJn+xPYXDlS11dCx+CJjTZMjey+++6xY+65557hz3/+c9yp2slu15ECoHiDjwCEn2/+2mTL0lcnu/hmz56dGcfGSPb1118/+CC0WUKk6Iceeqg247SB21e+8pWsjxmh89H5ZKH1wdFO6JAEUV9sfLZ6TJkyJRx55JFxiZHnleOcc86JEJ5++umZ6g4qO3bw7Vm+GsTBvf333z/qnRGPb30jTR921w1Cx5zJXEn/QzfPHDj94Ac/iCoDqA20exx11FHh2Wefjckh+eMrUNY2nO27wXa92267hYMOOihKky6++OIofexkLLZ69PKMelQnGKHnbypTZf0YnHgxLHJshuHLDtY2SOLsmo8Q2McDsFm73nrrhZVWWmnEhL2vhA69Ajrkv/zLv4R3vetd4f3vf3/cWMCGC+5rJ2hRd9A9EGCCZYLgaxw33XRTR6CwFMJAyMM0bdq0uGzeUYIDGPmpp56KOjxgJtcaAhASI2xjndBdeOGFGRaGyUjOTIY4iAO61osvvnhduksssURYa621wtprrx0Pywd95EmTJsXnfZhf9o3QUT+ey5E49DzRy+bTkOyaNYf6ieE1kjObGpFA43gBRO+P+TlN673vfW/cAEkdIBzmv9pqq0WyglBmUB1k2Mo70rNJRNGH/NCHPhT15NO0MGFW1E/5GtY73vGOujKwwZS0jNCBHysFhL2zREkAACAASURBVEVa1+zjDEV495XQUQDYPJ0axgvQnDkovJZei5pI9wwBJLxyrSHAM8ZgLULXGl6EMuV1BuoiQuf9R7uE7kc/+lE2AX3iE58IELxrrrkm/OEPfwinnnpq5gdWP/zhD+MqC5875FOQSN0gafhBEnCM+5i9YuJHWmKTITqsvMjPmTMnW73h60OYhKL/Em7llVcOkO1uzg/93KlshG6kOnR8ExzJDYIPpJbeISS58847I7bsqDVcwe7aa6+NtlFvvPHGGI+lPFYnTjrppLiJj7Djx4+P8zBpQuz4EtTtt98eN/JZWuiQsrJmbcSzQZpIcLF0QbiPfvSjXVddaccmrMek6JoyU34Ik9WL82WXXRZ++9vfRvz+9re/RckYGJx99tmBDydYWLDEIc2kH4OH2Zi1MJy99NTKAQFnExUvNxBiwtFWcB5Pzi38t771rRiG9Nvt830ndFZonYcDgXZ3lLJEgui4nd2jDPboyfGwcO7U8cbDQGdLCe0+FGn+pEe5WMa0NNMwRf/bxa4ojZHeY6NHPwldu3UFS5b1it5oy+oMSbV+gs5JI0f/o93oW83CWjpIkWxwLiJ03n+0EzozIM9SUdq2LC0ZTpyZKFOHXip+SNjS+CzhWnx0wso21F199dUBKR1hseV53nnnpdmM6D9jA23JsldathEl2CSSEbp111235b5oSTJ2sdwJBnzSrZGDqBmufE/dlgmL4kC+GR+Q+vnvjltYy5P0uC5z/iUHslLUF8riNroPYaLvmK3bRmHb8eO5NYw4I1Qqc4wdLKUSDoKXOuvHfO3K0txggw0apsmqEG3D2FfmeL6QgkLiIZntOBG6dtAa8LBMkuhB8HbL2xPSGXQimNzRd+BtGR0IvvDBfxO1W7WITyflO7LE/973vheVcBmQ2JSAOYeZM2cWdjK+O8tbNCZuMOVCJ2d5FFF1GaGC+LFkQCfnDW+zzTaL8fk4NG9BOCZw9BaoE+WyOqXEiredGTNmxDdL6syDyJsOb0ws51h6lN/qRx3Bo8zkDngwYGFaZ4sttgibbrpp1GOhzGn+kAfuW9qI3/kKCrvcrrvuuoDEY5999onSDAapRoOttUcn514SOqsr+DEpUldwRNLCpwG5vuSSS6JebFoH+glvp/Ql2oXlCCQ8ZfonxEcHhjwIu9FGG8XNCzvttFOc4O69995cFgzC559/fmz/rbbaKrAkxCYapEZ33HFHaV8kEU/Yxjqh4/lBypZKhMAJKZ1NYJyLJnHGDCRKjAc8R97ZREjcRoSOZ4w2tLwgFu28APg8/TUEknwZdwad0CEJW2GFFeJSdTOVpHYIHXhA1sG26CtGrRI6ns2ll146ayOkSt1wjF+rr756HF+6kZ6l0Q6hIw7zBRgxfqfO+jE62Ixl1k+ZL8oEEzxXELpmuqFGEpnD2nEidO2gNeBhGURZnrCOxRndBwjO/PPPn+ksmo7E9ttvH2655ZasVkxiaXzCQLT893IPOOCAaA/OIvKGx25l8sO0CJMn+pH8560NK+mp48Gyb1ci9seMiB9Ett566ziR82CnZeLN0hMqrg855JCY3yabbBK+853vRDK6+eabZ0sCZjfML/cYThDG1PGG+PnPfz6rEzgwCZgxacgF5MQchNHSszOTGflRvy233DISQvNDwXmk+jSWZ6MzyzRMqL1Yci2qK/hAnNAJoY6LLbZYVJ72dWRpaOedd47+RrIMj0996lPhrrvuqqsSS3S2u3T99dePJkTsP3Hpd0YYIM+27ES/o9+zXMQLBmHZAMNu1TLpsZc2FBE67z/aJXSQHV5gigyityKhoyFZqoWQpctKNhHSJo0IHUtufkxgCcz3p7rO0uIN2p8xal4QurKJvqzohx12WOy7vMg0UzlpRuh4EfPjJkvotAEvwqnzuHNd5kgP4kU6HLR5qxLxsjS5by+kvDB207VC6KgTUlwcQgfGja997Wt1xbB+jJQPHUfDAKknOtpFrlVCZ1+hAttG0rw0DxG6FJEh/s9ggYSOzmedizOKrugOIL3goebadB8gG+hV4IhPR/LxWXphqYA3RZ+mTXi8NTLY4Ec8iBCONzc6L/chQV4Cg44Gomn80CuxTQ6QECNM+P385z+PZcKwK/oH1IP7vOH4gQm9He4j5bMHkTLwFgSRws8IHZJJJGlMJNznSAkdZYeg4MckQrnMQV4tLm9RkGgc2CFZZKKwdD/5yU+GZZddNqATBPZMbN/85jcz/24tIVnZ/Jm692rJ1erK7i2rK0q84IKuid2DUBohgKwtssgi0Y+dkDbJI2EzPNGn8m+uSDYtLYz5moSCs5H85ZZbLmtbBr6llloqi3PppZdGSJhg2BlNWuRB/ytyXkJ366231gXx/tZn6wKNghtIwWgTnl8jy75arRI6MGdCYnL2ziZC2gOywJhU5CAwEH3rAxBM3z+K4rRyb1gIHWPZxz/+8Vh/XlyauZTQpeSXlQjGV1tm5HmhfXhRSV2rhI7+wRxibcQ44MfgNN1W/9v41WtCZ/OYlcsIHH0Ux+oSL6E8C6l02PoxeDIm2nwHFgg2DGdLmzNhWpHQMQctsMAC8TD9PZ9O2bUIXRkyQ3yfAZJOQ8diHR5CkToUQe0hpJNBOMwxEEAGzB8pD/680XKPh50HmU5s5mfGjRsXfvOb31gS8XzmmWdmabAciyMOD72lPX369CwODxMECD90ZvxyD5OyDTKe0PGQmY06pDOpg+CSnhE684fUWRk8oWMJBsKBH1KeIkkMgwBvbYShXl55F50Hw54HEgy8Y3nD8mVJtldLPjYg9kJCZ/WBWFldWZ5D6ZqBCBLNbkbqx4BIf2LJmnrTH1NJnE1E9KGrrroqJg8JsDi8fPhlPQg2X2YhPSZ5GzjJGyke95EQ+nwg4hB+/JCOFpEIL4FLB3oK5f2L+oXhMuxnyDbSTV5OilyrhA6dL14GUxJmEyFtAXEs06FD4movi4RFKt4N6Q9jEPn2UkLHuMTSP+MbqgmUHyzo6xyt2L5EXcVeUPw4WdQm3LPniLx4LlO1DtQW8LPnBSwZ+3lOU2djLeG5LnM8234HJysl3XDMBcxBvSZ0hoWV2eZFVEfMoULCak3a96wfWxqoIth4CG7EY0zyrlVCR17MP6RjO8V9OmXXInRlyAzxfSYrW5ZibT8dUKkaDzvLk3QYvr7gJRL4IV3Bj63VJnVDKkbntcmQCd2IDRMp9yE3dvCw27d/Wfpi4EYaSLocRUrCECIGGGz1+Dci0rZBhofGJHQQBnQDLU301CCCTMjE4S0UfS2rgzUr0jSL4wkdE79JD1lqZlJJHfUwHQfexPzSq+kOkTb4phM/ZMTy7WRCgVDz0JcdTLq0ITiWhbH7af1a/U9fsH4CDvZSgG4mfjZRQ/StzrwAWP+wM2W1fsISEyTXBlbiFRELln3pJ+nECPGz/mP5I+lhgrBda0wUvs2svp6wFRE6L6FL29XSGAvnVgldGRY2EdK2jQgdqhq2WxbVESS23XD0+14vuTJ2ob/JWMXEzMFqBP9ZafBjTlmd/HPD89DMNSJ0kHSTdhoBaZSejbW0USNC58dRFPlRq+iGmxeEjn5heoWe0JXVx/qxx5PNE3zyE9ze9ra3xR3hPr4RulRq7cPYtb3QoppUpoduYe0sQmdIjKIzD689kGWEjuqa3hudD1ME5iCANlFD9ozAmb+dUQYlLoct0dBh7YCwIJHBH+kIJITlU4uDJKdVx+RMHsT1hI746Zcf3ve+90UyRXg2KTABpc6Xww+uTBr2xsmya5mzsvDdXi9JZKI37JBEUmfvukHoILMHHnhgnJSYmIoOpBlMIvSDIn+7h55fKr305W10jQTB6kpblun4eKJkyxfWRzhzz0g0fRKiR7tZP/n2t7/dqBiFfuDM8jpvtxirpb0g6NZ/UukFiXjCVkTo/OQlQld57sHTS08LGyO5aRMhcel/tBUEnQOJBv2RlzSTwrJMyI7XoperJOmW/vaD0FEQ8rEXL15u+c/B80sfb+aQTtkzkG7+KYrrCR0vLUiV7DlHZ9XS8gSkKB3u2fxBHIQDqE5YG/Eiz4sv6iOMf4SBrNJG3XL9InSocvACCE4IHUwtZKSEjnnKDECDS7pTlfGO+asVQscYbvibAKMZviJ0zRAaQn8mKzoonaERobOlRcIx6Zmjs9lEveKKK5YqkdtyJvHJz0/S6TUTKoOb1yFjWadV14jQMVhC0Iw8Uh5/MDgxQHhXRujOPffcLC7b5sucH/CIY47BzrCjzilx8NKNkUroINhIIolfdrALlIEDjMvCcJ9Bv5XJwurnz76uSEHLnP8CA7ilfcP/Z7JHumGfBqQd21l2YZmVpXfTnSQ+7UH5rE+AS9oulN1PoEWEzpPMsUzosNNlWHJGv7Yd5wkd4xN6XEit2FDDjk6bVFEu56WIFyaTtraTT1nYfhG6svxbvc/mLvBlE5tJvxvF9YTOt0963S6hgxxy0EZsMEItBvUI0kW1gueX56Gb6iP9InQpNva/HUJnRoetbVgNstUv0mMesI0NYNUqobMxkJUF00W2PMrOInRlyAzxfVP6pTM1InSmKE44dq6ao/PwAHOfcyplsnAonROGo5E0y8JzNqOJxKGjt+ogMTwMxOOB8G8sSIZQxEVXkDA8ALZUY+VDZ847P3l7Cd1FF12U1YkdW2XOY3fBBRdkwbwOHSSXycM79MysTJCbMqmWjzOS637o0DGI0xbUh7fcsrrYMgbhMGXTivv617+e4cRO4VYckh7rI+SFVX128kGi0Zsy3CEQkNHUSUKXIlL837+UgGknEjrINmoB6JmBP2eIMy+LSOqKiHVxqWp3GSsgBGUH/QTpBy89ZWHsfqtLXeTOODnSo1b62pVN6JgFKRuDa6HzOnQs97JDnLaBcLDrEhUX2qtdQsdLIeMlbWNtRRtBIBnveKFu10F6DOOiMxJA6sDLWZG/v1c27hSViTHLxgHOvIyAE+MB5qSQ2HG/HUJXhCf6k34OMl1mxifqBW7NnFlkYNMJfbYVJ0LXCkpDFqYVQsfbFGY9rHP7DszgYVImJr+yt0PsetkmBj5sXrYkwtu17XziU1uWJ5277K2OCdi/lZdJ6MgTYgh5ssGXuAwyDEJmsgJ/X48yCR0PuNlVYhmwzKEzRj34hB3KsObQHzMyDGFLJUF+MkRCVlZ/S2+k534QOl9X2qBsGclLPTFPU+ZoY9Ob9O3TaIcffd36ibfczo5ru09+vABYv4OEsrTHC4DtUiYMy7MWpohImCkBwoxlCZ3vw2DRCaHjWbEdz2X9op379EH6IuMWaRcdjG2oI9APivztHhIpdDpbdTwP1n/aOZe9sNjSHZ9MY0xr5ryEjrqlY4+p2BQRkDRtvwLBON0OaUrTSv+zAsQ4YDgXnSE9tBFtVeRv98AGKwetupTQpVjY/OTnw7K0TdJcNFYwF/HNW+sH9CVeUMCStmllyZVNRcRncx1S8VacCF0rKA1ZGMgTBIbOUCaho3OhH0cYFI4hZ+Z4w+WBwY/O56VhFoYzb362OQA9l6KOTTgGLHarMSjx5mX5MugyCKYOQknH9xK8MkLHhM7gQz2L3npshxmbNrzY2hMGL6FjyQ5zI9SdN6NUwkZZwcdIH8u8/pt75MFARHzKn05WfjIcdkLHWza4W12NUKftyRuwEWs20aQTDeHps7QjmDB5sMxmb7gbbrhhYR8Ea/TisAEFgWfHK2XhSHdcQ3Dp5/gxSbDbmDL5ydSeGcIU9WXrS/iL0NXUGgaJ0NF3MGpNW5UdSAIZX5DSmeSpLKwn/Gm/Tv8zltGH2z18H/RpogZDX2MZ2l6IvX963YzQYSie9FISk6bDf+pAWA6w6iahYx5gzC3DnPsY8uU5RZeyUTjaMt3wVlQfu9eM0GHSiDpD1pq5RoSOuIxPEFfDEekf+oitEjrT2WXDWKv9UISuWasNoT8khCUwOhKKxX5J0KpjgwVheGggKeYgIUboOHs/C2Nnv2uVN8p04GFiZhKnc+KQSKHkbp2cQSaV7GEvCX9vnJEymE0mTzKpKwMO4SFpqTPpEMrXXmJjDyPxPKEjvm2y4M0IS+Gp85IcpDbeMagbocM4cvpm7QldtwdKX45+SOjQCzFC5yWkvhxc0+ZeX/OII45Ig0QChl4OSzk43660kZm98REhcsRhIwsvAd7EBUvn3vEMoJNFWrxQ0C8xh8JkYc7r+hUtyXoJXao3Y2mMhbPvw+DZCaFj0k5NO/QaQ/oW/bWXL1TdqINN6JjgaTQGW16e0IFr+lKCAWGWTFtJyxM6rste1izvbp9ZUmXugXB306WEjo193jH3MVaxoaqZszkkxdnHY2zydvqwg9kqoTMdSj5/JwmdR3WMXTO52QNpX4iA7EA26HxmTZ/BmLcGk2wxQaNrhu4Ryq74s3uJZQfuFxEm8mKgsK3avPVC8hjkmSxZtoS4eJtglMMke+SBKQsMtSK9Y7cthIgt9oSjTOySxMAxb6qE5wFhSY30kcxYXTEyi903JGa8HfEVDNvZaKQW8gaJtC8WkB46ItwzUxbUiU0H+EGIeXBJkzdBCJ4ZZUYCZ/ojhh3Gdj12LHMwiLIsy85UI9qkjdSQjRfoVxRJrTrptr0kdFZXNlSw5Exd+CIAgy9v1kWfUmOARhJBWJS8wYE2p+9Rf/R7aAcv+UJvxHa/Eo8XD/zpS7QJAyMTs71E0B6E46BPsKQKWaBPMTn4Psfy7IILLhhthbG0QduwAcjiky42tXghoN9j69BvuqGfoxvYin2wTtpxEON2k9DRLqkUu9d17jehYzzhmTF1glbrB6mw/tjKEp0ndDwbNja1mp8PZ2Mq+ffyxdPn6a/7RegakTFfnqJrI3SNNpXxMmuSUbBE2sZLcCvtaYbbMQdD/2nFSULXCkpDFobJ3IgD28qZjFgiZAnLyAhvcEhD/NsxhMWWxmwg8eeyjQ+QEfQYkKBh8oM8WU7lc2GInIukHZSRLyWwTMbESj7oTXBGgmdxIGZlZWJXHIMWxIIlXwgSdcQYp5E/CAEKqgziOBShfZ389Q033JC1NOkSD3MrGDkmXZbsuCYfJnL/UFJen5a/Jq59zcLft2vKPkyEDhJmZS86ly0hQdDBDf1DpGX0E/ohfZLdjEXLCvRJJKLooJAXAyL9irZA0ueXW2gD+g47XJFqEN7CQjhnzZqVSZ5Jh40X9ENv6DqtDyTV7Helfvw3yXPWccbAhQhda41M32JJEHtivLAwTvGyU6bCkqaKTTfrc14lJg1n/z2hgzSI0BkytXMqoes1oSNn5gnbbEF7tkrobFxijITgtuJE6FpBacjCoFdm0gomSyRfTGYQKKRdiIHpyEVidKQfSEY4kHDYNXpQzQYidDMYeMgHPRaMFTP4N3KEwWgmbztIz5Da8EbrHWUgf8rGUhn/mbyZzFmuPeOMM6J0DfJHeuRPehgmhST4jQc8XKTDwZuV1Q8JGrilDsKLuBsJHwfXRcSDeJTR0kNKyYBMeCR/pO39kR6af1l6aVna+c9kgvTDdle1E7eVsL4uaT8x8lyUDm0BHrQjE9Bll10W27zRGyhtTJ/lO6z0X5biwbeovQgL5vQpJGvslmQQtzLRR0iH9rd+RnmoD88JYVlORcLMPfKgn3l/CC3+pFFUhqJ6j6Z74GNEgzPt346D5Fj80Sqho8+g1sILK6sbfKXBlt5QTfEvg2XYMUbbSy6fa2zmbInWSIN/2WkWN/VHWm5tNJokdJ4kUz+e4ZE4CJap3qD60cxh3cA+V9gKoWOcRHeYMrJa0KoToWsVqSEKxxIGa/V0Btg9k5Tc2EIAQodUi+VeOSHQCQK8PCGdZ2Ln8LvjGWOQfk6o2hdE6btohyD30GMlvqlBEBd9RtQfMOqKTls/CHI/llx5maB+vKSajjCqKdzj8LYry9oGqb2tKECCixwqJoQBO2/7jDywvUi7gC+foWrmeAkmLG1kKxykw5If6aOy0A65aJZfI/9uLrkiaAAL6gUe1gacWVVC+IEfR7NlcfoxEntWfswWH1jRp8HGb5BL62cEsBVCh/DE2qCs7dP0+S9CV4TKkN9D6RUdCjosb8CmGzbk1VLx20CAXWlIKJEYygmBThBgRzzqDUxE/uBl0f/nGjMSqc1H8kayih/jkh3E5xopFOkz6Y4WQodOJuMvqi4mfeaZBAPuY2OuaIXEtxNSGrPHyJJdkWOjDi9uhilnw5Vrxn/+o/PZzKEqQVv4tIjPf+6j+gJh74frNqEr63tWVzDipaIZoaNve7w91vThRoQOs1m0Azg2k9Ci/kM/QYXJvm/dCu4idK2gNERhkMzwMLCZgQ5Bh8VEgyb2IWpEFVUIDBACEAukdIwrjC9MRhyoLHC2MQfigjTElrd9FdhhjvoFel12oGpg10yEpN8P1w8JHSoC73nPeyKh8vYv99prrzguY7TcJHeN6oxawlvf+tYorQHn1IErS7OGI2faxf83lY80bvqftGgDH9f/p/1sA1Iat9v/yRcy2a1drr6vFWHE/FiEb1ov2oylbMPIY93KHEtfaKWfmwF+Vtp8/0nLk/4XoUsRGeL/KJ3D/jG3AZmzAwVx7ttu1iGuooouBISAEOgIAQgdy7/oiUFWe+GY+FOSysTMV2wYl9mx3YptNwgzUjHitKJH14u6zIs0IT3MWWNRZQTSbGoJ7MBvx4nQtYPWgIeF0DFQsZbPgW4AOgMciOw72dEz4FVX8YSAEBACLSMA2eq3KgobxSBmfF2n2WYxXxHisbseYteKVM/HHdZriDaf7fNWGIa1Lu2Wmw1XWGoosxDRKD0RukboyE8ICAEhIASEQIcIQEwwXwKhY4d3K9I5y5LlPbNawMYFudGLABsa2ZiCTUwM3LfrROjaRUzhhYAQEAJCQAi0iADLpijDQ+ZGukqCqRj0ojFl0Y50r8UiKtiAIICZKfoJxvzNrFI7RROhawcthRUCQkAICAEh0CIC6M0dcMABcbcrCvQ4jIynnxtsJTlsdPJlFtRnWlHgbyVNhRkcBLDniBkfVKRGutQsQjc47amSCAEhIASEwChBAPNRfEIQO27oN5vjs3PoOLez7Epcwl999dXx6ycHHXSQJafzKECAHbJI5lhab2W3bFmVRejKkNF9ISAEhIAQEAIjQAATIJie4LvU2IqbNm1aPPj0F/b2OjHHgXkYyKLc6EKAnb2dftdYhG509QnVRggIASEgBOYxApdffnlYZJFFotQFyUt6QPDkhEC3ERCh6zaiSk8ICAEhIATGNAKTJk0Ke+yxR8CQcHqwrIYBZjkh0G0EROi6jajSEwJCQAgIgTGNwNy5c0OzY0wDpMr3BAERup7AqkSFgBAQAkJACAgBIdA/BETo+oe1chICQkAICAEhIASEQE8QEKHrCaxKVAgIASEgBISAEBAC/UNAhK5/WCsnISAEhIAQEAJCQAj0BAERup7AqkSFgBAQAkJACAgBIdA/BETo+oe1chICQkAICAEhIASEQE8QEKHrCaxKVAgIASEgBISAEBAC/UNAhK5/WCsnISAEhIAQEAJCQAj0BAERup7AqkSFgBAQAkJACAgBIdA/BETo+oe1chICQkAICAEhIASEQE8QEKHrCaxKVAgIASEgBISAEBAC/UNAhK5/WCsnISAEhIAQEAJCQAj0BAERup7AqkSFgBAQAkJACAgBIdA/BETo+oe1chICQkAICAEhIASEQE8QEKHrCaxKVAgIASEgBISAEBAC/UNAhK5/WCsnISAEhIAQEAJCQAj0BAERup7AqkSFgBAQAkJACAgBIdA/BETo+oe1chICQkAICAEhIASEQE8QEKHrCaxKVAgIASEgBISAEBAC/UNAhK5/WCsnISAEhIAQEAJCQAj0BAERup7AqkSFgBAQAkJACAgBIdA/BETo+oe1chICQkAICAEhIASEQE8QaIvQzZ07NzzwwAPh4Ycfbno8+OCD4YUXXgiXXnppOPvss8P5558fj2nTpoVzzjknnHvuueGaa64Jd999d3j22WfDm2++2ZMKKlEhIASEgBAQAkJACIx2BNoidBC5+eabLx4LL7xwGD9+fPaf+2uuuWZYcskls3tnnXVWOOmkk8Iaa6wR1llnnXifa4611lorhifevvvuG+67777RjrXqJwSEgBAQAkJACAiBniDQFqG7/vrrIyk788wzw3PPPRePb3zjG/HeHnvsER555JHw2GOPhVNOOSXeQxL32muvRUkd/pC33XffPTz11FPh5ZdfjucrrrgiksCll146/PWvf+1JJZWoEBACQkAICAEhIARGMwJtEboLL7wwrLfeeuH555/PMDn99NMjUdt7773DG2+8kd3fbbfdwtSpU7P/++yzTww3adKk7B4XEL4dd9wx+h177LHh9ddfz/nrjxAQAkJACAgBISAEhEBjBNoidOi/7bDDDrkU0Y9D8pYSuoMPPjjst99+WVgjdITzDt25L3/5yxnZe/XVV733UF3fPmV8WG7K7+dhmX8fJi8/PizHseuM8ESuJE+EC3YdHybfmrs5lH+emDmxVr9bp9SuQwg5v6GsnQotBISAEBACQqB9BNoidNOnTw8QM+/KCB3SvL322isLaoQuldARYNNNN42E7he/+EUWfhgv5jmhS8hNHsPRQ+hy9WpY51xI/RECQkAICAEhMGoRaIvQFaFQRujSsEboJk6cGB599NGoa8dO2NNOOy0suuii4dOf/nR46KGH0mgd/kdiNTFcMHNKRWoVpVdTwu2W6qMzwq74P2o3QvCkLEp7pvy+cq8q+ULCFe+bJMxJ5Cpxp9SkZEnaIVRIVZSgpVI0yrLrjHABUj78XLq10iV5L1+rS65My6eSuHy+u858IoSC/OqlWxWJn5fq5fPJY+fLGUIV+1vBuExq6CSKy48PsVxZInk/j0dWTsicpV3FIvMraNusTNbeufgpZuX5Z0XUhRAQAkJACAiBs0U2HgAAF+5JREFUAUGgY0Lndega6b8ZobNdrquttlpYYIEFwkYbbRTNmrCDtqFLJt/aRJ5OxD6V6qScLT9WiY2RpYJJv47QOXKUkZmS+DGuCx9imY30VPL2pMXnFQlWHanxdTEyl5K42v+YX1bXfFwjkxk5i3XPk6iMDGVRK/hZnDr/WD+XfxaPCyNENf9Y36x8Ff8aHv5/ilX+f64clCFL0y+55uPEovmwadlzfSGNm/7PVVR/hIAQEAJCQAjMcwQ6JnTtSujQl3viiSeiPbt//OMf4fHHHw8vvfRSj4DIExIyyZGB3CReKYInWbmweNeFr0z0Rnh83Epqjgh4MlHxrJKeKuGrSzsLVL2or4uRtIwUFeZh6eTLWl+XBJsYzefJtZFTS7Mi0czyr93OCJ1hE71cHeuwJUBGshxuuTQrf3Jxkzp7P39NTNqnUtYEi2oeMXwk643zLyiSbgkBISAEhIAQmKcI9J3QpZsielt7T0gqOeUmeUcwrByelOXCEiCGr0mcjFAZaakRBkuttoQb08qWB6tLkPF/i4SuoKzkksszITe1UnCVkJiC9OrqW5WyxfrF8L7ctetyQpcSwFp7eJyzcnp8I7krXqrNlTOpc84vV0dPSCvl8FLe7Nqkrw3yz8qrCyEgBISAEBACA4LA8BA6P8EmxMgIVT2mNQJhfuUTfiWEJxq5sHh7whGD50lSjlxVM7T06tKq+menHPnI7tYuSvxzeSbkphaZq3xZ25bQleSfz8P/8wTK7tfaw3Axn3iuw9cwrxK7KtnKYZnUOedXrXMknLlwtXLk8i/6E8uUz78omO4JASEgBISAEJiXCAwPoRsRSvUTd27CLyApnmjkwpJ/HeHIkyQft1Lcin9GKNwmhrrqFJQlH6a+LkbSMglZjrTkY1vYjPwW5FdXXy+hy12naRf9Lyivy7M+L7/kWpAedUs3PhAsqXOarv1ns0mGkyd6BVkV3nL5F/rrphAQAkJACAiBeYjAiAgdX4k4+eSTw6mnnhoOO+ywaHKETQ/sWD3uuOPi91utThdddFEMiwkTs1c3ZcqUgFmT3rt6UmETfMVGW8U/m+gj4ajtMM2HbZHQOT2zGD8jcRVy53dr1nTGLO10iTKPUD69+k0SKblJYuft0DlylYWLpKVWhkp+tU0naf628SEjiVlCXFSw9RsW8oQ3wb4avtIWjghX0/RxYzlsI0QTQlch4RUJW66cSV2N8LaSf66a+iMEhIAQEAJCYAAQGBGhYzPDLrvsEjBBsueee0ajwpz5/6UvfSlggNjcGWecETbeeOOwxRZbhK222ipsvvnm0e6ct1FnYbt/bkboTCpU09W6HaO1RUt7FC6SoGY6dDMamC2pkpxsydilVUSwCgAxklXR+XLxCZuQmzR6Fpf6leQXiVO1fLvO/H2eBFZ19jJ9M7+jN83MJHpTnGkRI2FZ2DweGbHGP5av2i6Ux8XNETojjlUinfcjoSqRzoh1lnne/Ey6w7hB/rUUdCUEhIAQEAJCYDAQGBGho+hz586Nn/riSw988svOmC7h2hzXdlg4/hNfbjQjUE+mR3NtVTchIASEgBAQAvMSgRETunlZaOU9DAiI0A1DK6mMQkAICAEhMDoQEKEbHe04gLUQoRvARlGRhIAQEAJCYJQiIEI3ShtW1RICQkAICAEhIATGDgIidGOnrVVTISAEhIAQEAJCYJQiIEI3ShtW1RICQkAICAEhIATGDgIidGOnrVVTISAEhIAQEAJCYJQiIEI3ShtW1RICQkAICAEhIATGDgIidGOnrVVTISAEhIAQEAJCYJQiIEI3ShtW1RICQkAICAEhIATGDgIidGOnrVVTISAEhIAQEAJCYJQiIEI3ShtW1RICQkAICAEhIATGDgIidGOnrVVTISAEhIAQEAJCYJQiIEI3ShtW1RICQkAICAEhIATGDgJtEbrzzz8/nH322eHUU08NJ5xwQulx/PHHh7lz54bLLrssnHXWWeG8886Lx7Rp0+J/7l111VXhzjvvDI888kh4+eWXxw7iqqkQEAJCQAgIASEgBLqMQFuEbsKECWGttdYKK620UphvvvnC2muvHdZYY4145nrZZZeN9/F74oknwsknnxz911lnnXifsKuvvnpYc80143nxxRcPO+ywQ7j55pvDSy+91OWqKTkhIASEgBAQAkJACIwNBNoidK+88kp4/PHHw1577RUJGpK4p556KpK3559/Pvz9738PU6ZMiX5I3l577bXwwgsvhFNOOSXe22OPPcJzzz0XJXLPPPNMuOeee8Imm2wS/a655pqxgbhqKQSEgBAQAkJACAiBLiPQFqEjb0japEmTIglj+TV1Tz75ZFhhhRWi1M38CIfUbu+99w5vvPGG3Y7nyZMnR79dd901PP300zk//RECQkAICAEhIASEgBBojkBHhO7000/PcjjnnHPCfvvtF958882wzTbbxOVW8/SE7vXXX7fb8YwuHmRv3XXXDZDBYXa3HT4zLHX4nHlYhTlh6riZYSmOCbPD42lJbplV8Rs3M0ycWeebC/34zCuK08iFGqQ/lbpPvYUyPR4umTAzVK757/0GqcwqixAQAkJACAiB7iDQEaHzEroDDzwwbLHFFrFU3J8+fXpWQk/oUgnd/vvvHwndpptuGli2HWY3zwkdhK2IyFVBpXzNiJzhP3yEzkrOOSV03k/XQkAICAEhIARGHwIdEbqjjz462M5VNjtstdVWhQgZodttt93CY489Fh599NF4Pvfcc8PGG28cNtpoo3DTTTcVxu3sJpKZK8IlM2uSqaXGzQq3WaIPzw4T8X/YboTgSVkkNYfPqdyrSr6Q+sT7JglzErlK3Fk1KVmSthGNKEFLpWiUZcLscAlSPvxcurXSJXm7uuTKNM5Lp4hdIThZvhbPSewqfjVsUkIX62Z1tvhZwZxksEHZY0mQ/BVgmiVFaQlTklfez7edSeHyda0QWPPLt6/lGdPM8M7HTwlyYxwsRZ2FgBAQAkJACPQXgY4IHbtW2bHKcinLprvsskth6Y3QsUP2wx/+cFh11VXDggsuGHfFfv/73w/33XdfYbzczVunhOWWH194TL41F9L9qRKNTGpVnaxt8m6F0DlylJGJkvg22WdLfZEwGemo5O0lZBUCWF2ijWVpLEGr5J+QLk+u2pHQ5coGZFWsqnWLeVVx89eEzP+vkaUK8PX1dA2SkbUMh2q9DbOGdYxlrtU/5P77clTKYGla3eL/XJxYG7c8W19230b5eqc4+FrqWggIASEgBIRAfxHoiNChN8cyKZsZdtxxx7jpoaj4RuiQ0D388MPh7rvvjjtjH3roobgLtihOd+75Sb6SYm5SboXQZWQwhFAXPk8c/ORfzS2ShUheIBI+rRiA8lUJX13aKQL1dTHJW0aOCvOopUP5srC129mVL7/HyV9ngasX0c8IrnnGujjiZffryGDFo5ZvkzrWkTGXcE5PLt8uOUKXC2dtWi1rIX61NmqEgy+JroWAEBACQkAI9BuBjgjdlVdemZUXEybYlCtyRuj22WefuEu2KExv7tUThNykXECiauSiQAJTR1TyxKGIMFl6Md9sGbG6rBr/t0joCsoKZrk8CwlJDdlcWLsd03XlKZDQGSGqLIOaxLGSQKxfYb0aELqEAGZt0rSOFbxtObYmgaMsvq3z7ZL3yy+7xrx9nQvrYnWu5FGEg8GpsxAQAkJACAiBeYFAR4Tud7/7XVZm7M1xFDkjdNive/HFF4uCNL/XwZKrn/gz8kCOBQTCCBjeubBZeE9U8sShiDBZenVppTUuKEsuSIl/Ls92CF1Mr0LkDB8ra2HdY2E8oarg4OPkylvyxxMoC5Jh00odq5Fivp4Qt0Hoaku1+fbLymEFKz3X41AaVB5CQAgIASEgBPqAQEeE7o477mipiEboxoKELr+ZoTLxZ0uuXt8tRa6EzNSCeQmU3XXpc6sNQldErDw5a0xuamVpHM7KWTsXha/lW0vXxagtW9duVq98/X3cyn0jqqmELvs/k00xjqCDn/9fl196w+eZ+um/EBACQkAICIH+IdAWoUNn7rTTTov25tgEAUE788wz4/dZiz7ddfHFF8fvvbLxwWzN7bnnnuHYY4+N9up6X836CTdPKCr+mV6ZSa38EpzXe4v+jgAk5jEqUiNbnqtK+DKCUCEZOcLnCURTQpemV/+/bULn6hZxQeJVUPc8ZlXimNUrwbBIsuka2vLJyFZS74p/DePcf48Xaebi+rZuRuiqy67U12FgOomGQSy2y7MxDq6SuhQCQkAICAEh0GcE2iJ0u+++e/jc5z4Xtttuu7DllltGu3ObbbZZvFdE6M4444zo95nPfCZsu+22YfPNNw+EJx0MEPfe+Um+klvxpFzVIZswO9xWNatB6LqwLRC6iTNnNzBbUimP6YDlpEE5clKOTIXgmM5bjfjEGJCPHEHJpwPhzMhrasrk8DnVHaiVNPN1r5LRTL+sRlpjDrHsVqaUJOXLUEl3Vpg6oRY+I3fVoI3qWCHNRXHzbZ2lEQlq3i9mE4max8PK2aCNUsxsQ4tF1VkICAEhIASEwDxCoC1Cx1ceIGIcGAi26/TrD1YX80/Dz50714LoPMYQyBPFMVZ5VVcICAEhIASEQI8QaIvQ9agMSnYMISBCN4YaW1UVAkJACAiBviEgQtc3qJURCIjQqR8IASEgBISAEOg+AiJ03cdUKQoBISAEhIAQEAJCoK8IiND1FW5lJgSEgBAQAkJACAiB7iMgQtd9TJWiEBACQkAICAEhIAT6ioAIXV/hVmZCQAgIASEgBISAEOg+AiJ03cdUKQoBISAEhIAQEAJCoK8IiND1FW5lJgSEgBAQAkJACAiB7iMgQtd9TJWiEBACQkAICAEhIAT6ioAIXV/hVmZCQAgIASEgBISAEOg+AiJ03cdUKQoBISAEhIAQEAJCoK8IiND1FW5lJgSEgBAQAkJACAiB7iMgQtd9TJWiEBACQkAICAEhIAT6ikBbhO78888P06ZNC6eddlo44YQTSo/jjjsuvP766+Hcc88NxLHjuuuuy1XusssuC2effXY477zzwoUXXhjOOuuscOWVV+bC6I8QEAJCQAgIASEgBIRAYwTaInQTJkwIa621Vlh//fXDfPPNF6/5b8cSSywR76+55prhn//8Z9h1113DhhtuGLbddtt4/2Mf+1h45plnshKdeuqpgbCkxbHSSiuF6dOnZ/66EAJCQAgIASEgBISAEGiOQFuE7pVXXgmPPfZYWHfddSMBO+WUU8LTTz8dnnzyyfDUU0+Fhx56KEyZMiX6P/fcc4HwELiNNtooI26HH354ePXVV2PJOBNv4sSJUdrH/7lz5zYvtUIIASEgBISAEBACQkAIZAi0ReiIBUkzQseSauruvffesPLKK4cnnngier388sthgw02CDNnzgyLLbZYWGWVVcIf//jHXLTdd989LuXmbuqPEBACQkAICAEhIASEQEsIdEToTj/99CyTPfbYI5x44olRWrfooouGBx98MPpB6FhWve+++8JRRx0VJXs777xzbul1tBC62w6fGZY6fE6GSf8v5oSp42aGpTgmzA6PpwW4ZVbFb9zMMHFmnW8u9OMzryhOIxdqyP5Qf4fLqKzjkDWJiisEhIAQEALdQaAjQmcSujfeeCMuqx577LFxMwTEjqVYHIRu1VVXjcux999/f9hll10iqTvyyCMD8XAidBGGzn8SwpImCOFsRuQszqgkO03wsbrrLASEgBAQAkJg2BDoiNBNnTo17mSdPHlyJGnsbk0dhI7NDujX4W6//fYwfvz48IEPfCDccsst8V7vCB0SqyvCJTNrkqmlxs0Kt1khH54dJuL/sN0IwUvZIqk5fE7lXlXyNfWWEOJ9k4Q5iVwl7qyalCxJO4THwyUTqhK0VIpGWSbMDpcg5cPPpVsrXZK3q0uuTONmBspZc0m+Fs9J7GKedj9U83HSrFg3q7MLV8nDSQYblJ2wFUxn53BISWa+Lvn2CbHNDMNZ4ZJUklhWp9z9Sh+I+VDHgn4QQrXvWN/IxU/wzZWpddJcax9dCQEhIASEgBDoDIGOCN0aa6wRl1PXWWedhoRumWWWCejW4V577bW4cYJdrTvuuGN44IEH4qYITJY0dLdOCcstP77wmHxrWcwq0ciISZXYGFkqmMjrCJ0jRxnRKIlvpCcjU5EEGCGp5O3Ji8/LiIr3T2tVyb9GSNP/gfyyuqaxK2Q1Sz9XNsJWsarWLSM7RsJcut7P4mV1rpLWLJ+kGJUyO0KUlCOfdgixThmBrJQxS9uIlJUtScvKlpHjBJ9aXvVtk8MyplvDvdJW1q6VMtXqn/5PANBfISAEhIAQEAI9QKAjQnfBBReE559/Pu58XXHFFUOZhG7JJZfMdOqowz333BO22267SAKR7vVWQufIQ0pOWiF0RhYoeF34ChGwyTxH0GJjOaKQkIlKWzL5V4lBXdqVELXfIqLg0idgYR61FChfRoZqt7MrX/4a2amX1mURDE8juOYR6+IIkN238B7TKpGsYOjwcHGychfUL5Y5l56LGPIS1xSfRnXM8qwSVGtjSz3GjfUuahcLpbMQEAJCQAgIgf4g0BGhu+qqq2Ip33zzzYCNuiIpG0uu7G61TRJWrWuuuSYsvvjiAendF7/4xR7tcq2fbP0kXk/Q8gQgF5aC1xGVekKXEiYjSTGtbMnSlgw5t0joSghfjXiMkNDFdF15CiR0maQrlt8kU5WWjPUrrNcICF1aFpcuuNZIlPWiErKZpmOEMyGEufbN4euJZaUPVZakHU6UzWNlZbW8akXUlRAQAkJACAiBniPQEaG78cYbswK+8MIL4cUXX8z+2wWEbqGFFqojdCy9HnTQQVFKt9NOOzUndB0suXrpSvkkXimxETD+5cJyI076nqi0SegaSJKKyKVhGM85wlHzGTGhc6TH8GlY95hlpb5e387HqZWq/KoOUy+hK6mjpRbjJoQpl16TOjWS0Jl+YyTkOeJX/1Jg5ak7E69K7FJiXxdWN4SAEBACQkAIdBGBjgjdXXfd1bQoL730Uph//vkDO1xTh0Fivh6BPl1vvhBRPxnXE4ACiZOXvHgSFglDY0JnUptKXd2SaJzsfdwEjSZkxqRkRr7q0udGjogk6VeXH41oFJEjT85yONUlVcO1cbi6iPUk2RO63HV93KL6xTJX26hZndL4adntPxtTDKcc0SsoUtEtS6exYZiimLonBISAEBACQmBkCLRF6DBTwndczbDwpEmT4jIr32JNv/CASZJzzjknHHrooWG55ZaLn/864ogjAl+Q8G7GjBnzjtBVCUQ2eZuEpxNCZ0uoJuHLFPqr0i0vYfIkrymhq0oMs/Tq/6eExePMNeTH6pqSjvg/XUb0RMkTW1/uFEOrtw/vCpLmmxLVSjk88a2RRwtrdahINWs299K00zql+KThs/TcRphY9FhfT/wdUa8jod7PVVyXQkAICAEhIAR6iEBbhG7PPfcMX/jCF+KGhq222ipsscUWYfPNN4+7VIsI3WabbRbY9HDppZeGa6+9Nlx00UXR8LCvD0uvfPrLbNp5v86vPRmopFY3icfJuqobNWF2uA0zGB0QuokzZzcwW1Ipjy3LNTOhUlT/jKTEpT1PfNqT0JnkKSvL4XMq0rMqYczjVCWjpifmSGssoxFh8y8hc4TNp8ud+jaKUjdLKyVXubyqZksykpyUM6mT5WV6i/VlsfgJrlZuV6aMVFIF34cIk5UnoqMfISAEhIAQEAI9R6AtQvf6668HNkBwIIGz65TMWaktjD+bnz/jzyEnBNpFIJIyEah2YVN4ISAEhIAQGGUItEXoRlndVZ1hQyC31Evh66V7w1YllVcICAEhIASEQDcQEKHrBopKo28INFyO7VsplJEQEAJCQAgIgcFCQIRusNpDpRECQkAICAEhIASEQNsIiNC1DZkiCAEhIASEgBAQAkJgsBAQoRus9lBphIAQEAJCQAgIASHQNgIidG1DpghCQAgIASEgBISAEBgsBEToBqs9VBohIASEgBAQAkJACLSNgAhd25ApghAQAkJACAgBISAEBgsBEbrBag+VRggIASEgBISAEBACbSMgQtc2ZIogBISAEBACQkAICIHBQkCEbrDaQ6URAkJACAgBISAEhEDbCIjQtQ2ZIggBISAEhIAQEAJCYLAQ+H9XR+qTHId5mgAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> ==> On utilise le score F1<b>\n",
    "    ![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> Un score F1 plus grand implique une précision ET recall plus grands<b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bruit uniforme à 30%\n",
    "X_dirty_2,y_dirty_2=UniformNoise(noise_level=0.3 # Bruit 30%\n",
    "                             ,random_state=1).fit_transform(X_clean.copy(),y_clean.copy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Division Train/Test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Division en Train & Test stratifiée, on n'utilise pas la cv sur du \"unsupervised learning\"\n",
    "# Dirty\n",
    "X_train_d, X_test_d, y_train_d, y_test_d = train_test_split(X_dirty_2, y_dirty_2,stratify=y_dirty_2, test_size=0.5, random_state=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean\n",
    "X_train_c, X_test_c, y_train_c, y_test_c = train_test_split(X_clean, y_clean,stratify=y_clean, test_size=0.5, random_state=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modèles basés sur les Ensembles & Combination "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Isolation Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Renvoie le score d'anomalie de chaque échantillon en utilisant l'algorithme IsolationForest\n",
    "\n",
    "La forêt d'isolement \"isole\" les observations en sélectionnant au hasard une caractéristique, puis en choisissant au hasard une valeur fractionnée entre les valeurs maximale et minimale de la caractéristique sélectionnée.\n",
    "\n",
    "Comme le partitionnement récursif peut être représenté par une structure arborescente, le nombre de fractionnements requis pour isoler un échantillon est équivalent à la longueur du chemin du nœud racine au nœud terminal.\n",
    "\n",
    "Cette longueur de chemin, dont la moyenne est calculée sur une forêt de tels arbres aléatoires, est une mesure de normalité et notre fonction de décision.\n",
    "\n",
    "Le partitionnement aléatoire produit des chemins sensiblement plus courts pour les anomalies. Par conséquent, lorsqu'une forêt d'arbres aléatoires produit collectivement des chemins plus courts pour des échantillons particuliers, il est très probable qu'il s'agisse d'anomalies.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.547\n"
     ]
    }
   ],
   "source": [
    "isof = IsolationForest(contamination=0.3)\n",
    "isof.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = isof.predict(X_test_d)\n",
    "# On code les inliers 1, outliers -1\n",
    "y_test_bi=LabelEncoder().fit_transform(y_test_c)\n",
    "y_test_bi[y_test_bi == 1] = -1\n",
    "y_test_bi[y_test_bi == 0] = 1\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extended Isolation Forest"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "@ARTICLE{8888179,\n",
    "author={S. {Hariri} and M. {Carrasco Kind} and R. J. {Brunner}},\n",
    "journal={IEEE Transactions on Knowledge and Data Engineering},\n",
    "title={Extended Isolation Forest},\n",
    "year={2019},\n",
    "volume={},\n",
    "number={},\n",
    "pages={1-1},\n",
    "keywords={Forestry;Vegetation;Distributed databases;Anomaly detection;Standards;Clustering algorithms;Heating systems;Anomaly Detection;Isolation Forest},\n",
    "doi={10.1109/TKDE.2019.2947676},\n",
    "ISSN={},\n",
    "month={},}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>J'ai utilisé une regression logistique pour définir une frontière de décision entre les probabilités définies par l'EIF<b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_ex,X_train_lr=X_train_d[:int(len(X_train_d)*0.5)],X_train_d[int(len(X_train_d)*0.5):]\n",
    "X_test_ex,X_test_lr=X_test_d[:int(len(X_test_d)*0.5)],X_test_d[int(len(X_test_d)*0.5):]\n",
    "ex_isof=eif.iForest(X_train_ex, ntrees=200,sample_size=165, ExtensionLevel=2)\n",
    "X_train_l=ex_isof.compute_paths(X_in=X_train_lr)\n",
    "X_test_l=ex_isof.compute_paths(X_in=X_test_d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.567\n"
     ]
    }
   ],
   "source": [
    "lr=LogisticRegression(C=0.6,random_state=1)\n",
    "lr.fit(X_train_l.reshape(-1,1),y_train_d[int(len(X_train_d)*0.5):].reshape(-1,1))\n",
    "yhat=lr.predict(X_test_l.reshape(-1,1))\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_c, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.246\n"
     ]
    }
   ],
   "source": [
    "fb=FeatureBagging(LOF(n_neighbors=57, contamination=0.3),\n",
    "                       contamination=0.3,\n",
    "                       random_state=1)\n",
    "fb.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = fb.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSCP-Locally Selective Combination "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.266\n"
     ]
    }
   ],
   "source": [
    "detector_list = [LOF(n_neighbors=5), LOF(n_neighbors=10), LOF(n_neighbors=15),\n",
    "                 LOF(n_neighbors=20), LOF(n_neighbors=25), LOF(n_neighbors=30),\n",
    "                 LOF(n_neighbors=35), LOF(n_neighbors=40), LOF(n_neighbors=45),\n",
    "                 LOF(n_neighbors=50),LOF(n_neighbors=57),LOF(n_neighbors=67),LOF(n_neighbors=75)]\n",
    "\n",
    "lscp=LSCP(detector_list, contamination=0.3,random_state=1)\n",
    "lscp.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = lscp.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.345\n"
     ]
    }
   ],
   "source": [
    "detector_list = [LOF(n_neighbors=57),OCSVM(contamination=0.3),]\n",
    "\n",
    "lscp=LSCP(detector_list, contamination=0.3,random_state=1)\n",
    "lscp.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = lscp.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modèles Linéaires"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA- Principal Component Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PCA utilise la somme des distances projetées pondérées par rapport à l'hyperplan du vecteur propre comme scores des valeurs aberrantes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.254\n"
     ]
    }
   ],
   "source": [
    "pca = PCA(contamination=0.3,random_state=1)\n",
    "pca.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = pca.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MCD-Minimum Covariance Determinant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Utilise la distance de Mahalanobis(robustesse à la covariance des features) pour le scoring des outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.258\n"
     ]
    }
   ],
   "source": [
    "mcd = MCD(contamination=0.3,random_state=1)\n",
    "mcd.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = mcd.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OCSVM- One-Class Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.324\n"
     ]
    }
   ],
   "source": [
    "ocsvm = OCSVM(contamination=0.3)\n",
    "ocsvm.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = ocsvm.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LMDD-Linear Model Deviation-base outlier detection "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "’aad’: Average Absolute Deviation\n",
    "\n",
    "’var’: Variance\n",
    "\n",
    "’iqr’: Interquartile Range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.196\n"
     ]
    }
   ],
   "source": [
    "lmdd= LMDD(contamination=0.3, n_iter=50, dis_measure='aad', random_state=1)\n",
    "lmdd.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = lmdd.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.199\n"
     ]
    }
   ],
   "source": [
    "lmdd= LMDD(contamination=0.3, n_iter=50, dis_measure='var', random_state=1)\n",
    "lmdd.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = lmdd.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.080\n"
     ]
    }
   ],
   "source": [
    "lmdd= LMDD(contamination=0.3, n_iter=50, dis_measure='iqr', random_state=1)\n",
    "lmdd.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = lmdd.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modèles basés sur la proximité"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOF-Local Outlier Factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.242\n"
     ]
    }
   ],
   "source": [
    "lof = LOF(n_neighbors=57, contamination=0.3)\n",
    "lof.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = lof.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CBLOF-Clustering-Based Local Outlier Factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.307\n"
     ]
    }
   ],
   "source": [
    "cblof = CBLOF(random_state=1,check_estimator=False, contamination=0.3)\n",
    "cblof.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = cblof.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Median kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.293\n"
     ]
    }
   ],
   "source": [
    "mknn = KNN(method='median', contamination=0.3)\n",
    "mknn.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = mknn.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.282\n"
     ]
    }
   ],
   "source": [
    "aknn = KNN(method='mean', contamination=0.3)\n",
    "aknn.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = aknn.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HBOS-Histogram-based Outlier Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.273\n"
     ]
    }
   ],
   "source": [
    "hbos = HBOS(contamination=0.3)\n",
    "hbos.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = hbos.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOD-Subspace Outlier Detection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.292\n"
     ]
    }
   ],
   "source": [
    "sod = SOD(contamination=0.3,n_neighbors=57, ref_set=3, alpha=0.8)\n",
    "sod.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = sod.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modèles probabilistes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ABOD: Angle-Based Outlier Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=X_train_ex.copy()\n",
    "X = X.astype('float64') # Problème avec dtype \n",
    "abod = ABOD(contamination=0.3)\n",
    "abod.fit(X)\n",
    "\n",
    "Xt=X_test_d.copy().astype('float64')\n",
    "Xc=X_train_lr.copy().astype('float64')\n",
    "X_train_ll=abod.decision_function(X_train_lr)\n",
    "X_test_ll=abod.decision_function(X_test_d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.517\n"
     ]
    }
   ],
   "source": [
    "lr=LogisticRegression(C=0.6,random_state=1)\n",
    "X2=SimpleImputer(missing_values=np.nan,strategy=\"median\").fit_transform(X_train_ll.reshape(-1, 1))\n",
    "X3=SimpleImputer(missing_values=np.nan,strategy=\"median\").fit_transform(X_test_ll.reshape(-1, 1))\n",
    "lr.fit(X2.reshape(-1,1),y_train_d[int(len(X_train_d)*0.5):].reshape(-1,1))\n",
    "yhat=lr.predict(X3.reshape(-1,1))\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_c, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### COPOD - Copula Based Outlier Detector "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.264\n"
     ]
    }
   ],
   "source": [
    "mad = COPOD(contamination=0.3)\n",
    "mad.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = mad.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOS-Stochastic Outlier Selection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.303\n"
     ]
    }
   ],
   "source": [
    "sos = SOS(contamination=0.3,perplexity=35, metric='euclidean', eps=2)\n",
    "sos.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = sos.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single-Objective Generative Adversarial Active Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 of 90\n",
      "\n",
      "Testing for epoch 1 index 1:\n",
      "Epoch 2 of 90\n",
      "\n",
      "Testing for epoch 2 index 1:\n",
      "Epoch 3 of 90\n",
      "\n",
      "Testing for epoch 3 index 1:\n",
      "Epoch 4 of 90\n",
      "\n",
      "Testing for epoch 4 index 1:\n",
      "Epoch 5 of 90\n",
      "\n",
      "Testing for epoch 5 index 1:\n",
      "Epoch 6 of 90\n",
      "\n",
      "Testing for epoch 6 index 1:\n",
      "Epoch 7 of 90\n",
      "\n",
      "Testing for epoch 7 index 1:\n",
      "Epoch 8 of 90\n",
      "\n",
      "Testing for epoch 8 index 1:\n",
      "Epoch 9 of 90\n",
      "\n",
      "Testing for epoch 9 index 1:\n",
      "Epoch 10 of 90\n",
      "\n",
      "Testing for epoch 10 index 1:\n",
      "Epoch 11 of 90\n",
      "\n",
      "Testing for epoch 11 index 1:\n",
      "Epoch 12 of 90\n",
      "\n",
      "Testing for epoch 12 index 1:\n",
      "Epoch 13 of 90\n",
      "\n",
      "Testing for epoch 13 index 1:\n",
      "Epoch 14 of 90\n",
      "\n",
      "Testing for epoch 14 index 1:\n",
      "Epoch 15 of 90\n",
      "\n",
      "Testing for epoch 15 index 1:\n",
      "Epoch 16 of 90\n",
      "\n",
      "Testing for epoch 16 index 1:\n",
      "Epoch 17 of 90\n",
      "\n",
      "Testing for epoch 17 index 1:\n",
      "Epoch 18 of 90\n",
      "\n",
      "Testing for epoch 18 index 1:\n",
      "Epoch 19 of 90\n",
      "\n",
      "Testing for epoch 19 index 1:\n",
      "Epoch 20 of 90\n",
      "\n",
      "Testing for epoch 20 index 1:\n",
      "Epoch 21 of 90\n",
      "\n",
      "Testing for epoch 21 index 1:\n",
      "Epoch 22 of 90\n",
      "\n",
      "Testing for epoch 22 index 1:\n",
      "Epoch 23 of 90\n",
      "\n",
      "Testing for epoch 23 index 1:\n",
      "Epoch 24 of 90\n",
      "\n",
      "Testing for epoch 24 index 1:\n",
      "Epoch 25 of 90\n",
      "\n",
      "Testing for epoch 25 index 1:\n",
      "Epoch 26 of 90\n",
      "\n",
      "Testing for epoch 26 index 1:\n",
      "Epoch 27 of 90\n",
      "\n",
      "Testing for epoch 27 index 1:\n",
      "Epoch 28 of 90\n",
      "\n",
      "Testing for epoch 28 index 1:\n",
      "Epoch 29 of 90\n",
      "\n",
      "Testing for epoch 29 index 1:\n",
      "Epoch 30 of 90\n",
      "\n",
      "Testing for epoch 30 index 1:\n",
      "Epoch 31 of 90\n",
      "\n",
      "Testing for epoch 31 index 1:\n",
      "Epoch 32 of 90\n",
      "\n",
      "Testing for epoch 32 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6488\n",
      "Epoch 33 of 90\n",
      "\n",
      "Testing for epoch 33 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6454\n",
      "Epoch 34 of 90\n",
      "\n",
      "Testing for epoch 34 index 1:\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6430\n",
      "Epoch 35 of 90\n",
      "\n",
      "Testing for epoch 35 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6427\n",
      "Epoch 36 of 90\n",
      "\n",
      "Testing for epoch 36 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6457\n",
      "Epoch 37 of 90\n",
      "\n",
      "Testing for epoch 37 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6366\n",
      "Epoch 38 of 90\n",
      "\n",
      "Testing for epoch 38 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6375\n",
      "Epoch 39 of 90\n",
      "\n",
      "Testing for epoch 39 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6422\n",
      "Epoch 40 of 90\n",
      "\n",
      "Testing for epoch 40 index 1:\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6297\n",
      "Epoch 41 of 90\n",
      "\n",
      "Testing for epoch 41 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6321\n",
      "Epoch 42 of 90\n",
      "\n",
      "Testing for epoch 42 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6378\n",
      "Epoch 43 of 90\n",
      "\n",
      "Testing for epoch 43 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6387\n",
      "Epoch 44 of 90\n",
      "\n",
      "Testing for epoch 44 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6425\n",
      "Epoch 45 of 90\n",
      "\n",
      "Testing for epoch 45 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6380\n",
      "Epoch 46 of 90\n",
      "\n",
      "Testing for epoch 46 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6447\n",
      "Epoch 47 of 90\n",
      "\n",
      "Testing for epoch 47 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6310\n",
      "Epoch 48 of 90\n",
      "\n",
      "Testing for epoch 48 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6425\n",
      "Epoch 49 of 90\n",
      "\n",
      "Testing for epoch 49 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6495\n",
      "Epoch 50 of 90\n",
      "\n",
      "Testing for epoch 50 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6327\n",
      "Epoch 51 of 90\n",
      "\n",
      "Testing for epoch 51 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6360\n",
      "Epoch 52 of 90\n",
      "\n",
      "Testing for epoch 52 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6285\n",
      "Epoch 53 of 90\n",
      "\n",
      "Testing for epoch 53 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6407\n",
      "Epoch 54 of 90\n",
      "\n",
      "Testing for epoch 54 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6441\n",
      "Epoch 55 of 90\n",
      "\n",
      "Testing for epoch 55 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6292\n",
      "Epoch 56 of 90\n",
      "\n",
      "Testing for epoch 56 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6289\n",
      "Epoch 57 of 90\n",
      "\n",
      "Testing for epoch 57 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6416\n",
      "Epoch 58 of 90\n",
      "\n",
      "Testing for epoch 58 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6419\n",
      "Epoch 59 of 90\n",
      "\n",
      "Testing for epoch 59 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6480\n",
      "Epoch 60 of 90\n",
      "\n",
      "Testing for epoch 60 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6338\n",
      "Epoch 61 of 90\n",
      "\n",
      "Testing for epoch 61 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6262\n",
      "Epoch 62 of 90\n",
      "\n",
      "Testing for epoch 62 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6346\n",
      "Epoch 63 of 90\n",
      "\n",
      "Testing for epoch 63 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6386\n",
      "Epoch 64 of 90\n",
      "\n",
      "Testing for epoch 64 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6298\n",
      "Epoch 65 of 90\n",
      "\n",
      "Testing for epoch 65 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6426\n",
      "Epoch 66 of 90\n",
      "\n",
      "Testing for epoch 66 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6309\n",
      "Epoch 67 of 90\n",
      "\n",
      "Testing for epoch 67 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6408\n",
      "Epoch 68 of 90\n",
      "\n",
      "Testing for epoch 68 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6363\n",
      "Epoch 69 of 90\n",
      "\n",
      "Testing for epoch 69 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6410\n",
      "Epoch 70 of 90\n",
      "\n",
      "Testing for epoch 70 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6465\n",
      "Epoch 71 of 90\n",
      "\n",
      "Testing for epoch 71 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6323\n",
      "Epoch 72 of 90\n",
      "\n",
      "Testing for epoch 72 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6309\n",
      "Epoch 73 of 90\n",
      "\n",
      "Testing for epoch 73 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6214\n",
      "Epoch 74 of 90\n",
      "\n",
      "Testing for epoch 74 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6373\n",
      "Epoch 75 of 90\n",
      "\n",
      "Testing for epoch 75 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6372\n",
      "Epoch 76 of 90\n",
      "\n",
      "Testing for epoch 76 index 1:\n",
      "11/11 [==============================] - 0s 3ms/step - loss: 0.6272\n",
      "Epoch 77 of 90\n",
      "\n",
      "Testing for epoch 77 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6294\n",
      "Epoch 78 of 90\n",
      "\n",
      "Testing for epoch 78 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6401\n",
      "Epoch 79 of 90\n",
      "\n",
      "Testing for epoch 79 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6320\n",
      "Epoch 80 of 90\n",
      "\n",
      "Testing for epoch 80 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6447\n",
      "Epoch 81 of 90\n",
      "\n",
      "Testing for epoch 81 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6362\n",
      "Epoch 82 of 90\n",
      "\n",
      "Testing for epoch 82 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6224\n",
      "Epoch 83 of 90\n",
      "\n",
      "Testing for epoch 83 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6363\n",
      "Epoch 84 of 90\n",
      "\n",
      "Testing for epoch 84 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6291\n",
      "Epoch 85 of 90\n",
      "\n",
      "Testing for epoch 85 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6225\n",
      "Epoch 86 of 90\n",
      "\n",
      "Testing for epoch 86 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6355\n",
      "Epoch 87 of 90\n",
      "\n",
      "Testing for epoch 87 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6286\n",
      "Epoch 88 of 90\n",
      "\n",
      "Testing for epoch 88 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6404\n",
      "Epoch 89 of 90\n",
      "\n",
      "Testing for epoch 89 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6377\n",
      "Epoch 90 of 90\n",
      "\n",
      "Testing for epoch 90 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6386\n",
      "F1 Score: 0.263\n"
     ]
    }
   ],
   "source": [
    "sog=SO_GAAL(stop_epochs=30, lr_d=0.0001, lr_g=0.00001, decay=0.7, contamination=0.3)\n",
    "sog.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = sog.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.263\n"
     ]
    }
   ],
   "source": [
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple-Objective Generative Adversarial Active Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 of 90\n",
      "\n",
      "Testing for epoch 1 index 1:\n",
      "WARNING:tensorflow:5 out of the last 29 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDC3FF040> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 31 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDCE8A0D0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDC3FFC10> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 12 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDCE8A4C0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDC2B40D0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 12 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDC1EF160> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDBDD98B0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:9 out of the last 12 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001FEDC477AF0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 124 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDD190550> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 125 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDC438A60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 126 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDCFDE670> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 127 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDD06D280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:9 out of the last 128 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDBDD9DC0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:10 out of the last 129 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDC2B4040> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 130 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDC161790> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_train_function.<locals>.train_function at 0x000001FEDBC21430> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "Epoch 2 of 90\n",
      "\n",
      "Testing for epoch 2 index 1:\n",
      "Epoch 3 of 90\n",
      "\n",
      "Testing for epoch 3 index 1:\n",
      "Epoch 4 of 90\n",
      "\n",
      "Testing for epoch 4 index 1:\n",
      "Epoch 5 of 90\n",
      "\n",
      "Testing for epoch 5 index 1:\n",
      "Epoch 6 of 90\n",
      "\n",
      "Testing for epoch 6 index 1:\n",
      "Epoch 7 of 90\n",
      "\n",
      "Testing for epoch 7 index 1:\n",
      "Epoch 8 of 90\n",
      "\n",
      "Testing for epoch 8 index 1:\n",
      "Epoch 9 of 90\n",
      "\n",
      "Testing for epoch 9 index 1:\n",
      "Epoch 10 of 90\n",
      "\n",
      "Testing for epoch 10 index 1:\n",
      "Epoch 11 of 90\n",
      "\n",
      "Testing for epoch 11 index 1:\n",
      "Epoch 12 of 90\n",
      "\n",
      "Testing for epoch 12 index 1:\n",
      "Epoch 13 of 90\n",
      "\n",
      "Testing for epoch 13 index 1:\n",
      "Epoch 14 of 90\n",
      "\n",
      "Testing for epoch 14 index 1:\n",
      "Epoch 15 of 90\n",
      "\n",
      "Testing for epoch 15 index 1:\n",
      "Epoch 16 of 90\n",
      "\n",
      "Testing for epoch 16 index 1:\n",
      "Epoch 17 of 90\n",
      "\n",
      "Testing for epoch 17 index 1:\n",
      "Epoch 18 of 90\n",
      "\n",
      "Testing for epoch 18 index 1:\n",
      "Epoch 19 of 90\n",
      "\n",
      "Testing for epoch 19 index 1:\n",
      "Epoch 20 of 90\n",
      "\n",
      "Testing for epoch 20 index 1:\n",
      "Epoch 21 of 90\n",
      "\n",
      "Testing for epoch 21 index 1:\n",
      "Epoch 22 of 90\n",
      "\n",
      "Testing for epoch 22 index 1:\n",
      "Epoch 23 of 90\n",
      "\n",
      "Testing for epoch 23 index 1:\n",
      "Epoch 24 of 90\n",
      "\n",
      "Testing for epoch 24 index 1:\n",
      "Epoch 25 of 90\n",
      "\n",
      "Testing for epoch 25 index 1:\n",
      "Epoch 26 of 90\n",
      "\n",
      "Testing for epoch 26 index 1:\n",
      "Epoch 27 of 90\n",
      "\n",
      "Testing for epoch 27 index 1:\n",
      "Epoch 28 of 90\n",
      "\n",
      "Testing for epoch 28 index 1:\n",
      "Epoch 29 of 90\n",
      "\n",
      "Testing for epoch 29 index 1:\n",
      "Epoch 30 of 90\n",
      "\n",
      "Testing for epoch 30 index 1:\n",
      "Epoch 31 of 90\n",
      "\n",
      "Testing for epoch 31 index 1:\n",
      "Epoch 32 of 90\n",
      "\n",
      "Testing for epoch 32 index 1:\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.8247\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.7570\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7350\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.7217\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7135\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7059\n",
      "11/11 [==============================] - 0s 951us/step - loss: 0.6979\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6828\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6707\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6146\n",
      "Epoch 33 of 90\n",
      "\n",
      "Testing for epoch 33 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8443\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7652\n",
      "11/11 [==============================] - 0s 904us/step - loss: 0.7383\n",
      "11/11 [==============================] - 0s 804us/step - loss: 0.7230\n",
      "11/11 [==============================] - 0s 938us/step - loss: 0.7149\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.7055\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6954\n",
      "11/11 [==============================] - 0s 899us/step - loss: 0.6793\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6639\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6002\n",
      "Epoch 34 of 90\n",
      "\n",
      "Testing for epoch 34 index 1:\n",
      "11/11 [==============================] - 0s 895us/step - loss: 0.8373\n",
      "11/11 [==============================] - 0s 899us/step - loss: 0.7613\n",
      "11/11 [==============================] - 0s 899us/step - loss: 0.7345\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.7207\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7131\n",
      "11/11 [==============================] - 0s 895us/step - loss: 0.7038\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6933\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6791\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6625\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6030\n",
      "Epoch 35 of 90\n",
      "\n",
      "Testing for epoch 35 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.8414\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.7620\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7314\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7181\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7103\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7022\n",
      "11/11 [==============================] - 0s 797us/step - loss: 0.6912\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6767\n",
      "11/11 [==============================] - 0s 904us/step - loss: 0.6583\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.5983\n",
      "Epoch 36 of 90\n",
      "\n",
      "Testing for epoch 36 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8357\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7590\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7294\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7161\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7086\n",
      "11/11 [==============================] - 0s 893us/step - loss: 0.7017\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6908\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.6751\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6595\n",
      "11/11 [==============================] - 0s 904us/step - loss: 0.6022\n",
      "Epoch 37 of 90\n",
      "\n",
      "Testing for epoch 37 index 1:\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.8336\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7567\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7281\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7147\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7078\n",
      "11/11 [==============================] - 0s 896us/step - loss: 0.7007\n",
      "11/11 [==============================] - 0s 899us/step - loss: 0.6900\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.6733\n",
      "11/11 [==============================] - 0s 995us/step - loss: 0.6593\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6030\n",
      "Epoch 38 of 90\n",
      "\n",
      "Testing for epoch 38 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.8311\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7540\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7271\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7136\n",
      "11/11 [==============================] - 0s 904us/step - loss: 0.7076\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7002\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6896\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6734\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6587\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.6046\n",
      "Epoch 39 of 90\n",
      "\n",
      "Testing for epoch 39 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.8388\n",
      "11/11 [==============================] - 0s 901us/step - loss: 0.7559\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.7285\n",
      "11/11 [==============================] - 0s 901us/step - loss: 0.7140\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.7081\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7002\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.6892\n",
      "11/11 [==============================] - 0s 906us/step - loss: 0.6718\n",
      "11/11 [==============================] - 0s 901us/step - loss: 0.6555\n",
      "11/11 [==============================] - 0s 895us/step - loss: 0.5992\n",
      "Epoch 40 of 90\n",
      "\n",
      "Testing for epoch 40 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8307\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7505\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7242\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7102\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.7052\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6974\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6870\n",
      "11/11 [==============================] - 0s 899us/step - loss: 0.6701\n",
      "11/11 [==============================] - 0s 896us/step - loss: 0.6537\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6003\n",
      "Epoch 41 of 90\n",
      "\n",
      "Testing for epoch 41 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.8339\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7515\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7249\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7107\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7058\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6977\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6872\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6694\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6522\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5985\n",
      "Epoch 42 of 90\n",
      "\n",
      "Testing for epoch 42 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8358\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.7518\n",
      "11/11 [==============================] - 0s 998us/step - loss: 0.7251\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7108\n",
      "11/11 [==============================] - 0s 901us/step - loss: 0.7059\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6976\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6871\n",
      "11/11 [==============================] - 0s 800us/step - loss: 0.6692\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6509\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.5972\n",
      "Epoch 43 of 90\n",
      "\n",
      "Testing for epoch 43 index 1:\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.8424\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7537\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7259\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7111\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7057\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6971\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6863\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.6675\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6475\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.5920\n",
      "Epoch 44 of 90\n",
      "\n",
      "Testing for epoch 44 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.8409\n",
      "11/11 [==============================] - 0s 999us/step - loss: 0.7517\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7241\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.7094\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7038\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6944\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6847\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6660\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6452\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.5903\n",
      "Epoch 45 of 90\n",
      "\n",
      "Testing for epoch 45 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.8482\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7539\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7254\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7107\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7047\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6943\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6845\n",
      "11/11 [==============================] - 0s 999us/step - loss: 0.6656\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6431\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.5865\n",
      "Epoch 46 of 90\n",
      "\n",
      "Testing for epoch 46 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8288\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7458\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7204\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7079\n",
      "11/11 [==============================] - ETA: 0s - loss: 0.702 - 0s 1ms/step - loss: 0.7028\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6937\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6847\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6686\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6482\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5989\n",
      "Epoch 47 of 90\n",
      "\n",
      "Testing for epoch 47 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8317\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7467\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7210\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7083\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7028\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6935\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6846\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6685\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6464\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5974\n",
      "Epoch 48 of 90\n",
      "\n",
      "Testing for epoch 48 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8339\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7461\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7199\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7069\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7011\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6910\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6824\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6662\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6422\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.5934\n",
      "Epoch 49 of 90\n",
      "\n",
      "Testing for epoch 49 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8412\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7489\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7214\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7079\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7018\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6911\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6812\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6654\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6398\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5892\n",
      "Epoch 50 of 90\n",
      "\n",
      "Testing for epoch 50 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8338\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7443\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7179\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7042\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6990\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6887\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6783\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6640\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6389\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5906\n",
      "Epoch 51 of 90\n",
      "\n",
      "Testing for epoch 51 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8335\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7449\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7191\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7057\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7005\n",
      "11/11 [==============================] - 0s 897us/step - loss: 0.6903\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6800\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6661\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6410\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5940\n",
      "Epoch 52 of 90\n",
      "\n",
      "Testing for epoch 52 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8312\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7434\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7183\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7050\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7000\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6899\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6790\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6662\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6411\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5953\n",
      "Epoch 53 of 90\n",
      "\n",
      "Testing for epoch 53 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8348\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7440\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7182\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7044\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6987\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6890\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6778\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6648\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6386\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5921\n",
      "Epoch 54 of 90\n",
      "\n",
      "Testing for epoch 54 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8271\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7405\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7161\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7029\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6975\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6884\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6778\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6655\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6402\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5964\n",
      "Epoch 55 of 90\n",
      "\n",
      "Testing for epoch 55 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8279\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7402\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7157\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7023\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6970\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6877\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6773\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6648\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6389\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5953\n",
      "Epoch 56 of 90\n",
      "\n",
      "Testing for epoch 56 index 1:\n",
      "11/11 [==============================] - 0s 3ms/step - loss: 0.8274\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7397\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7155\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7021\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6970\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6877\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6774\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6649\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6389\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.5960\n",
      "Epoch 57 of 90\n",
      "\n",
      "Testing for epoch 57 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8341\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7414\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7160\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7020\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6966\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6861\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6760\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6629\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6351\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5906\n",
      "Epoch 58 of 90\n",
      "\n",
      "Testing for epoch 58 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8384\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7431\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7171\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7030\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6973\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6859\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6759\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6629\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6340\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5890\n",
      "Epoch 59 of 90\n",
      "\n",
      "Testing for epoch 59 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.8464\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7450\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7171\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7028\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6968\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6845\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6738\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6605\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6294\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5827\n",
      "Epoch 60 of 90\n",
      "\n",
      "Testing for epoch 60 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8154\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7345\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7123\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7010\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6961\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6862\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6777\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6671\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6418\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6046\n",
      "Epoch 61 of 90\n",
      "\n",
      "Testing for epoch 61 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8369\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7413\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7152\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7020\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6962\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6845\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6746\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6622\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6322\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.5893\n",
      "Epoch 62 of 90\n",
      "\n",
      "Testing for epoch 62 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8326\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7394\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7140\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7013\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6957\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6842\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6747\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6626\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6331\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.5919\n",
      "Epoch 63 of 90\n",
      "\n",
      "Testing for epoch 63 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8400\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7421\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7154\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7021\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6962\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6842\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6741\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6617\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6303\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5876\n",
      "Epoch 64 of 90\n",
      "\n",
      "Testing for epoch 64 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8327\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7403\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7149\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7027\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6970\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6856\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6759\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6645\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6346\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5947\n",
      "Epoch 65 of 90\n",
      "\n",
      "Testing for epoch 65 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8331\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7381\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7119\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6996\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6936\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6820\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6717\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6604\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6294\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5889\n",
      "Epoch 66 of 90\n",
      "\n",
      "Testing for epoch 66 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8611\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7484\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7174\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7030\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6957\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6821\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6699\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6568\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6200\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.5730\n",
      "Epoch 67 of 90\n",
      "\n",
      "Testing for epoch 67 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8546\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7461\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7163\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7025\n",
      "11/11 [==============================] - 0s 996us/step - loss: 0.6950\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6824\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6707\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6581\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6225\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5776\n",
      "Epoch 68 of 90\n",
      "\n",
      "Testing for epoch 68 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8357\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7387\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7123\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7000\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6932\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6820\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6715\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6603\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6284\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5883\n",
      "Epoch 69 of 90\n",
      "\n",
      "Testing for epoch 69 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.8252\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7357\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7117\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7004\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6941\n",
      "11/11 [==============================] - 0s 997us/step - loss: 0.6838\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6742\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6639\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6345\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5976\n",
      "Epoch 70 of 90\n",
      "\n",
      "Testing for epoch 70 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8237\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7346\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7111\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7000\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6937\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6836\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6740\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6638\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6348\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5982\n",
      "Epoch 71 of 90\n",
      "\n",
      "Testing for epoch 71 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8210\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7329\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7100\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6992\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6929\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6830\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6736\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6635\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6350\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5990\n",
      "Epoch 72 of 90\n",
      "\n",
      "Testing for epoch 72 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8302\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7350\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7106\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6990\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6922\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6816\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6715\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6607\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6301\n",
      "11/11 [==============================] - 0s 3ms/step - loss: 0.5916\n",
      "Epoch 73 of 90\n",
      "\n",
      "Testing for epoch 73 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8453\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7397\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7131\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7003\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6928\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6811\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6700\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6582\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6245\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5822\n",
      "Epoch 74 of 90\n",
      "\n",
      "Testing for epoch 74 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.8512\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7412\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7135\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7003\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6926\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6804\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6689\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6566\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6216\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5778\n",
      "Epoch 75 of 90\n",
      "\n",
      "Testing for epoch 75 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8325\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7348\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7104\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6987\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6919\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6807\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6707\n",
      "11/11 [==============================] - 0s 3ms/step - loss: 0.6596\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6286\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5898\n",
      "Epoch 76 of 90\n",
      "\n",
      "Testing for epoch 76 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8360\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7368\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7120\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7002\n",
      "11/11 [==============================] - 0s 903us/step - loss: 0.6933\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6820\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6717\n",
      "11/11 [==============================] - 0s 997us/step - loss: 0.6602\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6289\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5898\n",
      "Epoch 77 of 90\n",
      "\n",
      "Testing for epoch 77 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8336\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7351\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7106\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6990\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6921\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6809\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6707\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6591\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6282\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5898\n",
      "Epoch 78 of 90\n",
      "\n",
      "Testing for epoch 78 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8429\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7384\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7125\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.7002\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6928\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6810\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6701\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.6576\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6252\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.5847\n",
      "Epoch 79 of 90\n",
      "\n",
      "Testing for epoch 79 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8387\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7363\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7110\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6990\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6917\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6802\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6694\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6570\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6253\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.5859\n",
      "Epoch 80 of 90\n",
      "\n",
      "Testing for epoch 80 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8374\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7360\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7111\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6993\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6920\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6807\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6700\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6575\n",
      "11/11 [==============================] - 0s 999us/step - loss: 0.6261\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5875\n",
      "Epoch 81 of 90\n",
      "\n",
      "Testing for epoch 81 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8357\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7348\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7100\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6983\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6910\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6798\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6691\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6564\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6251\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5871\n",
      "Epoch 82 of 90\n",
      "\n",
      "Testing for epoch 82 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.8366\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7362\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7116\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.7000\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6927\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6816\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6707\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6581\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6269\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5896\n",
      "Epoch 83 of 90\n",
      "\n",
      "Testing for epoch 83 index 1:\n",
      "11/11 [==============================] - 0s 900us/step - loss: 0.8407\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7367\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7114\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6994\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6918\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6804\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6686\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6559\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6235\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5854\n",
      "Epoch 84 of 90\n",
      "\n",
      "Testing for epoch 84 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8438\n",
      "11/11 [==============================] - 0s 997us/step - loss: 0.7372\n",
      "11/11 [==============================] - 0s 995us/step - loss: 0.7113\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6991\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6913\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6796\n",
      "11/11 [==============================] - 0s 953us/step - loss: 0.6670\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6542\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6211\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.5825\n",
      "Epoch 85 of 90\n",
      "\n",
      "Testing for epoch 85 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8434\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7363\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.7103\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6982\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6902\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6785\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6657\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6529\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6197\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5812\n",
      "Epoch 86 of 90\n",
      "\n",
      "Testing for epoch 86 index 1:\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.8433\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7364\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7105\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6985\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6905\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6789\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6661\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6531\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6203\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5819\n",
      "Epoch 87 of 90\n",
      "\n",
      "Testing for epoch 87 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8428\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7365\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7109\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6990\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6910\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6795\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6668\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6537\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6214\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5833\n",
      "Epoch 88 of 90\n",
      "\n",
      "Testing for epoch 88 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8386\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7349\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7104\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6989\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6910\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6799\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6676\n",
      "11/11 [==============================] - 0s 3ms/step - loss: 0.6546\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6234\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5863\n",
      "Epoch 89 of 90\n",
      "\n",
      "Testing for epoch 89 index 1:\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.8377\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7328\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7086\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6970\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6891\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6779\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6656\n",
      "11/11 [==============================] - 0s 1000us/step - loss: 0.6525\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6210\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.5839\n",
      "Epoch 90 of 90\n",
      "\n",
      "Testing for epoch 90 index 1:\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.8475\n",
      "11/11 [==============================] - 0s 999us/step - loss: 0.7355\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.7101\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6979\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6894\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6775\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.6644\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6506\n",
      "11/11 [==============================] - 0s 1ms/step - loss: 0.6169\n",
      "11/11 [==============================] - 0s 2ms/step - loss: 0.5778\n",
      "F1 Score: 0.284\n"
     ]
    }
   ],
   "source": [
    "mog=MO_GAAL(k=10, stop_epochs=30, lr_d=0.0001, lr_g=0.0001, decay=0.7, contamination=0.3)\n",
    "mog.fit(X_train_d)\n",
    "\n",
    "# On détecte les anomalies sur le test set\n",
    "yhat = mog.predict(X_test_d)\n",
    "\n",
    "# On score avec F1\n",
    "score = f1_score(y_test_bi, yhat,average=\"weighted\")\n",
    "print('F1 Score: %.3f' % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.284\n"
     ]
    }
   ],
   "source": [
    "print('F1 Score: %.3f' % score)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}