multiclass-tumor-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tumor Classification Using One-vs-one Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code below aims to classify 5 different classes of tumors based on cleaned tabular data with 800+ features. As there are very few samples for each class, here we adopted one-vs-one logistic regression with votings to do the classification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/andrewchen/Desktop/Tumor Classification (ML)/multiclass_classification/data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split, LeaveOneOut\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.externals import joblib\n",
    "import os\n",
    "os.chdir('./data')\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data and split into X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(136, 831)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_excel('Features of IA subtypes.xlsx')\n",
    "labels = ['Lepidic', 'Papillary', 'Acinar', 'Micropapillary', 'Solid']\n",
    "X = data.drop('Unnamed: 0', axis = 1)\n",
    "X_unlabeled = X.drop('IA_type', 1)\n",
    "Y = data['IA_type']\n",
    "X.to_csv('X.csv', header = True, index = False)\n",
    "Y.to_csv('Y.csv', header = True, index = False)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a Logistic Regression Classifier for each pair of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training class 1: 1\n",
      "Training class 2: 2\n",
      "Columns kept:  [  6  20  42  44  48  49  56  58  92 101 202 249 253 269 339 340 345 395\n",
      " 450 451 460 633 692 702 724 812 820]\n",
      "Score: 0.8571428571428571\n",
      "Training class 1: 1\n",
      "Training class 2: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [ 23  38  43  68 175 203 459 619 634 733]\n",
      "Score: 0.45454545454545453\n",
      "Training class 1: 1\n",
      "Training class 2: 4\n",
      "Columns kept:  [115 765]\n",
      "Score: 0.8\n",
      "Training class 1: 1\n",
      "Training class 2: 5\n",
      "Columns kept:  [115]\n",
      "Score: 0.8333333333333334\n",
      "Training class 1: 2\n",
      "Training class 2: 1\n",
      "Columns kept:  [  6  20  42  44  48  49  56  58  92 101 202 249 253 269 339 340 345 395\n",
      " 450 451 460 633 692 702 724 812 820]\n",
      "Score: 0.8571428571428571\n",
      "Training class 1: 2\n",
      "Training class 2: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [722]\n",
      "Score: 0.75\n",
      "Training class 1: 2\n",
      "Training class 2: 4\n",
      "Columns kept:  [188 819]\n",
      "Score: 0.5\n",
      "Training class 1: 2\n",
      "Training class 2: 5\n",
      "Columns kept:  [668 705 724 788 827]\n",
      "Score: 0.6666666666666666\n",
      "Training class 1: 3\n",
      "Training class 2: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [ 23  38  43  68 175 203 459 619 634 733]\n",
      "Score: 0.45454545454545453\n",
      "Training class 1: 3\n",
      "Training class 2: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [722]\n",
      "Score: 0.75\n",
      "Training class 1: 3\n",
      "Training class 2: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [  1   2   3   4   6   9  11  16  21  23  28  33  38  42  43  48  71  88\n",
      " 100 102 111 122 126 158 184 198 199 210 212 214 247 269 270 293 294 295\n",
      " 296 297 298 308 312 313 320 344 357 359 360 362 377 378 397 440 462 473\n",
      " 536 570 579 585 588 601 614 616 627 647 649 679 692 714 722 733 738 740\n",
      " 742 759 770 772 777 780 783 784 787 790 809 811 813 822]\n",
      "Score: 1.0\n",
      "Training class 1: 3\n",
      "Training class 2: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [826]\n",
      "Score: 0.8571428571428571\n",
      "Training class 1: 4\n",
      "Training class 2: 1\n",
      "Columns kept:  [115 765]\n",
      "Score: 0.8\n",
      "Training class 1: 4\n",
      "Training class 2: 2\n",
      "Columns kept:  [188 819]\n",
      "Score: 0.5\n",
      "Training class 1: 4\n",
      "Training class 2: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [  1   2   3   4   6   9  11  16  21  23  28  33  38  42  43  48  71  88\n",
      " 100 102 111 122 126 158 184 198 199 210 212 214 247 269 270 293 294 295\n",
      " 296 297 298 308 312 313 320 344 357 359 360 362 377 378 397 440 462 473\n",
      " 536 570 579 585 588 601 614 616 627 647 649 679 692 714 722 733 738 740\n",
      " 742 759 770 772 777 780 783 784 787 790 809 811 813 822]\n",
      "Score: 1.0\n",
      "Training class 1: 4\n",
      "Training class 2: 5\n",
      "Columns kept:  [ 23  38  41  43  85 114 115 154 159 166 168 305 383 404 411 439 442 469\n",
      " 486 534 538 567 608 610 611 614 615 616 647 651 712 713 721 722 738 741\n",
      " 746 772 782 783 813 815 818 827]\n",
      "Score: 1.0\n",
      "Training class 1: 5\n",
      "Training class 2: 1\n",
      "Columns kept:  [115]\n",
      "Score: 0.8333333333333334\n",
      "Training class 1: 5\n",
      "Training class 2: 2\n",
      "Columns kept:  [668 705 724 788 827]\n",
      "Score: 0.6666666666666666\n",
      "Training class 1: 5\n",
      "Training class 2: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/myenv/lib/python3.8/site-packages/numpy/lib/nanfunctions.py:1546: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns kept:  [826]\n",
      "Score: 0.8571428571428571\n",
      "Training class 1: 5\n",
      "Training class 2: 4\n",
      "Columns kept:  [ 23  38  41  43  85 114 115 154 159 166 168 305 383 404 411 439 442 469\n",
      " 486 534 538 567 608 610 611 614 615 616 647 651 712 713 721 722 738 741\n",
      " 746 772 782 783 813 815 818 827]\n",
      "Score: 1.0\n"
     ]
    }
   ],
   "source": [
    "# Create a list of class labels\n",
    "list = [1, 2, 3, 4, 5]\n",
    "# Create a list of tuples for binary classifier. e.g. (1, 2)\n",
    "class_pair_combo = [(x, y) for x in list for y in list if x != y]\n",
    "# Initiate random seed\n",
    "random_seed = 0\n",
    "# Initiate matrices for OVO result aggregation\n",
    "weight_matrix = []\n",
    "# Split the data into training and testing sets\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.1, random_state = random_seed)\n",
    "X_train.to_csv('X_train.csv', header = True, index = False)\n",
    "X_test.to_csv('X_test.csv', header = True, index = False)\n",
    "Y_train.to_csv('Y_train.csv', header = True, index = False)\n",
    "Y_test.to_csv('Y_test.csv', header = True, index = False)\n",
    "\n",
    "# Train different binary logistic regression classifiers\n",
    "for element in class_pair_combo:\n",
    "    # Update random seed to increate randomness\n",
    "    random_seed += 1\n",
    "    \n",
    "    # Find the class labels from the tuples\n",
    "    first_class = element[0]\n",
    "    second_class = element[1]\n",
    "    \n",
    "    # Set the name for the binary classification model\n",
    "    model_name = 'model_' + str(element[0]) + '_' + str(element[1]) + '.pkl'\n",
    "    \n",
    "    # Print out two classes\n",
    "    print(\"Training class 1: %d\" % first_class)\n",
    "    print(\"Training class 2: %d\" % second_class)\n",
    "    \n",
    "    # Filter the training sets to keep only the samples of the two classes\n",
    "    X_train_filtered = X_train.loc[X_train['IA_type'].isin(element)]\n",
    "    X_train_filtered_unlabeled = X_train_filtered.drop('IA_type', 1)\n",
    "    X_train_filtered_unlabeled = StandardScaler().fit_transform(X_train_filtered_unlabeled)\n",
    "    X_train_filtered_unlabeled = pd.DataFrame(data = X_train_filtered_unlabeled)\n",
    "    Y_train_filtered = X_train_filtered['IA_type']\n",
    "    \n",
    "    X_test_filtered = X_test.loc[X_test['IA_type'].isin(element)]\n",
    "    X_test_filtered_unlabeled = X_test_filtered.drop('IA_type', 1)\n",
    "    X_test_filtered_unlabeled = StandardScaler().fit_transform(X_test_filtered_unlabeled)\n",
    "    X_test_filtered_unlabeled = pd.DataFrame(data = X_test_filtered_unlabeled)\n",
    "    Y_test_filtered = X_test_filtered['IA_type']\n",
    "\n",
    "    # Remove outlier for X_train_filtered_unlabeled\n",
    "    Q1 = X_train_filtered_unlabeled.quantile(q = 0.25, axis = 0, numeric_only = False)\n",
    "    Q3 = X_train_filtered_unlabeled.quantile(q = 0.75, axis = 0, numeric_only = False)\n",
    "    lowerBound = Q1 - 1.5 * (Q3 - Q1)\n",
    "    upperBound = Q3 + 1.5 * (Q3 - Q1)\n",
    "    for index, row in X_train_filtered_unlabeled.iterrows():\n",
    "        for col in range(len(X_train_filtered_unlabeled.columns)):\n",
    "            if row[col] < lowerBound[col]:\n",
    "                row[col] = np.nan\n",
    "            elif row[col] > upperBound[col]:\n",
    "                row[col] = np.nan\n",
    "    Q2 = X_train_filtered_unlabeled.quantile(q = 0.50, axis = 0, numeric_only = False)\n",
    "    for index, row in X_train_filtered_unlabeled.iterrows():\n",
    "        for col in range(len(X_train_filtered_unlabeled.columns)):\n",
    "            if pd.isna(row[col]):\n",
    "                row[col] = Q2[col]\n",
    "                \n",
    "    # Remove outlier for X_test_filtered_unlabeled\n",
    "    Q1 = X_test_filtered_unlabeled.quantile(q = 0.25, axis = 0, numeric_only = False)\n",
    "    Q3 = X_test_filtered_unlabeled.quantile(q = 0.75, axis = 0, numeric_only = False)\n",
    "    lowerBound = Q1 - 1.5 * (Q3 - Q1)\n",
    "    upperBound = Q3 + 1.5 * (Q3 - Q1)\n",
    "    for index, row in X_test_filtered_unlabeled.iterrows():\n",
    "        for col in range(len(X_test_filtered_unlabeled.columns)):\n",
    "            if row[col] < lowerBound[col]:\n",
    "                row[col] = np.nan\n",
    "            elif row[col] > upperBound[col]:\n",
    "                row[col] = np.nan\n",
    "    Q2 = X_test_filtered_unlabeled.quantile(q = 0.50, axis = 0, numeric_only = False)\n",
    "    for index, row in X_test_filtered_unlabeled.iterrows():\n",
    "        for col in range(len(X_test_filtered_unlabeled.columns)):\n",
    "            if pd.isna(row[col]):\n",
    "                row[col] = Q2[col]\n",
    "    \n",
    "    loo = LeaveOneOut()\n",
    "    feature_rank = np.zeros(X_train_filtered_unlabeled.shape[1])\n",
    "\n",
    "    for train_index, test_index in loo.split(X_train_filtered_unlabeled):\n",
    "        x_train_loo, x_test_loo, y_train_loo, y_test_loo = X_train_filtered_unlabeled.iloc[train_index], X_train_filtered_unlabeled.iloc[test_index], Y_train_filtered.iloc[train_index], Y_train_filtered.iloc[test_index]\n",
    "\n",
    "        # Create the RFE object and compute a cross-validated score.\n",
    "        svc = SVC(kernel = \"linear\")\n",
    "        # The \"accuracy\" scoring is proportional to the number of correct classifications\n",
    "        rfecv = RFECV(estimator = svc, step = 1, cv = 3, scoring = 'accuracy').fit(x_train_loo, y_train_loo)\n",
    "        optimal_feature_num = rfecv.n_features_\n",
    "        feature_ranking = rfecv.ranking_\n",
    "        feature_ranking_filtered = np.where(feature_ranking <= 1, feature_ranking, 0)\n",
    "        feature_rank = np.add(feature_rank, feature_ranking_filtered)\n",
    "#         print(\"Optimal number of features : %d\" % optimal_feature_num)\n",
    "#         print(\"Feature Ranking            : \", feature_rank)\n",
    "\n",
    "    #keep_col_index = np.where(feature_rank > np.percentile(feature_rank, 95))[0]\n",
    "    keep_col_index = np.where(feature_rank > X_train_filtered_unlabeled.shape[0] * 0.5)[0]\n",
    "    print(\"Columns kept: \", keep_col_index)\n",
    "    \n",
    "    column_kept_file_name = 'col_kept_' + str(element[0]) + '_' + str(element[1]) + '_df.csv'\n",
    "    keep_col_index_df = pd.DataFrame(data = keep_col_index)\n",
    "    keep_col_index_df.to_csv(column_kept_file_name, header = False, index = False)\n",
    "\n",
    "    # Train logistic regression\n",
    "    lr = LogisticRegression(random_state = 0, solver = 'lbfgs')\n",
    "    \n",
    "    X_train_filtered_unlabeled_FS = X_train_filtered_unlabeled[X_train_filtered_unlabeled.columns[keep_col_index]]\n",
    "    X_test_filtered_unlabeled_FS = X_test_filtered_unlabeled[X_test_filtered_unlabeled.columns[keep_col_index]]\n",
    "\n",
    "    lr.fit(X_train_filtered_unlabeled_FS, Y_train_filtered)\n",
    "    print(\"Score:\", lr.score(X_test_filtered_unlabeled_FS, Y_test_filtered))\n",
    "    \n",
    "    joblib.dump(lr, model_name)\n",
    "    \n",
    "    weight_matrix.append(len(X_train_filtered))\n",
    "    \n",
    "weight_matrix = (np.array(weight_matrix)).reshape(5, 4)\n",
    "weight_matrix_df = pd.DataFrame(data = weight_matrix)\n",
    "weight_matrix_df.to_csv('weight_matrix.csv', header = False, index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the weight matrix to calculate the probability matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To aggregate the results of the OVO classifiers, we need to obtain the probability matrix M, where M<sub>ijk</sub> = the confidence of kth testing data being the class represented by i over the class represented by j. (i, j) represents the unique pair of classes such that class i is different from class j."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14, 831)\n",
      "(27, 1)\n",
      "[[9.90080524e-01 9.91947617e-03]\n",
      " [6.96117085e-01 3.03882915e-01]\n",
      " [9.99779192e-01 2.20808277e-04]\n",
      " [9.99988535e-01 1.14649837e-05]\n",
      " [7.68765044e-01 2.31234956e-01]\n",
      " [2.91072158e-01 7.08927842e-01]\n",
      " [4.13719129e-01 5.86280871e-01]\n",
      " [6.47009545e-02 9.35299045e-01]\n",
      " [8.30566255e-01 1.69433745e-01]\n",
      " [9.85945993e-01 1.40540070e-02]\n",
      " [9.95888230e-01 4.11176969e-03]\n",
      " [4.95987422e-02 9.50401258e-01]\n",
      " [9.99905232e-01 9.47683658e-05]\n",
      " [9.99696398e-01 3.03602065e-04]]\n",
      "(14, 831)\n",
      "(10, 1)\n",
      "[[8.92205694e-01 1.07794306e-01]\n",
      " [5.57684775e-02 9.44231523e-01]\n",
      " [9.39531634e-01 6.04683658e-02]\n",
      " [9.99829203e-01 1.70796712e-04]\n",
      " [2.19103839e-04 9.99780896e-01]\n",
      " [2.47127750e-02 9.75287225e-01]\n",
      " [1.02483835e-02 9.89751617e-01]\n",
      " [1.20346224e-04 9.99879654e-01]\n",
      " [5.51738186e-01 4.48261814e-01]\n",
      " [7.08426486e-01 2.91573514e-01]\n",
      " [7.49840591e-01 2.50159409e-01]\n",
      " [1.68689806e-02 9.83131019e-01]\n",
      " [9.98757323e-01 1.24267744e-03]\n",
      " [9.90866853e-01 9.13314655e-03]]\n",
      "(14, 831)\n",
      "(2, 1)\n",
      "[[9.05422907e-01 9.45770934e-02]\n",
      " [9.29369539e-01 7.06304612e-02]\n",
      " [9.21740106e-01 7.82598937e-02]\n",
      " [9.99783579e-01 2.16420922e-04]\n",
      " [5.36403251e-01 4.63596749e-01]\n",
      " [1.91892786e-01 8.08107214e-01]\n",
      " [5.14783887e-01 4.85216113e-01]\n",
      " [8.57307570e-01 1.42692430e-01]\n",
      " [8.61194797e-01 1.38805203e-01]\n",
      " [9.50048449e-01 4.99515511e-02]\n",
      " [9.39105501e-01 6.08944988e-02]\n",
      " [9.99645021e-01 3.54978837e-04]\n",
      " [9.73641571e-01 2.63584295e-02]\n",
      " [9.49423035e-01 5.05769645e-02]]\n",
      "(14, 831)\n",
      "(1, 1)\n",
      "[[9.70515887e-01 2.94841134e-02]\n",
      " [9.53212325e-01 4.67876750e-02]\n",
      " [9.70515887e-01 2.94841134e-02]\n",
      " [9.70515887e-01 2.94841134e-02]\n",
      " [3.80016387e-01 6.19983613e-01]\n",
      " [3.05630580e-01 6.94369420e-01]\n",
      " [5.33150127e-01 4.66849873e-01]\n",
      " [9.50548129e-01 4.94518713e-02]\n",
      " [8.87197186e-01 1.12802814e-01]\n",
      " [8.83540136e-01 1.16459864e-01]\n",
      " [9.70515887e-01 2.94841134e-02]\n",
      " [9.99975970e-01 2.40295886e-05]\n",
      " [9.70515887e-01 2.94841134e-02]\n",
      " [9.70515887e-01 2.94841134e-02]]\n",
      "(14, 831)\n",
      "(27, 1)\n",
      "(14, 831)\n",
      "(1, 1)\n",
      "[[0.22584962 0.77415038]\n",
      " [0.25979619 0.74020381]\n",
      " [0.22303861 0.77696139]\n",
      " [0.22302021 0.77697979]\n",
      " [0.23919035 0.76080965]\n",
      " [0.69108473 0.30891527]\n",
      " [0.23676787 0.76323213]\n",
      " [0.54750005 0.45249995]\n",
      " [0.2240069  0.7759931 ]\n",
      " [0.22349723 0.77650277]\n",
      " [0.22588996 0.77411004]\n",
      " [0.36748267 0.63251733]\n",
      " [0.22314582 0.77685418]\n",
      " [0.22301996 0.77698004]]\n",
      "(14, 831)\n",
      "(2, 1)\n",
      "[[0.54895079 0.45104921]\n",
      " [0.34276748 0.65723252]\n",
      " [0.80656877 0.19343123]\n",
      " [0.92830939 0.07169061]\n",
      " [0.23716781 0.76283219]\n",
      " [0.06329897 0.93670103]\n",
      " [0.06636335 0.93363665]\n",
      " [0.24332096 0.75667904]\n",
      " [0.60321927 0.39678073]\n",
      " [0.55370075 0.44629925]\n",
      " [0.97801712 0.02198288]\n",
      " [0.33153073 0.66846927]\n",
      " [0.80656877 0.19343123]\n",
      " [0.80656877 0.19343123]]\n",
      "(14, 831)\n",
      "(5, 1)\n",
      "[[0.05831576 0.94168424]\n",
      " [0.89026624 0.10973376]\n",
      " [0.62954882 0.37045118]\n",
      " [0.73981889 0.26018111]\n",
      " [0.04189802 0.95810198]\n",
      " [0.02247245 0.97752755]\n",
      " [0.3958769  0.6041231 ]\n",
      " [0.95725335 0.04274665]\n",
      " [0.88287626 0.11712374]\n",
      " [0.94139699 0.05860301]\n",
      " [0.87328735 0.12671265]\n",
      " [0.9201848  0.0798152 ]\n",
      " [0.82348744 0.17651256]\n",
      " [0.64522573 0.35477427]]\n",
      "(14, 831)\n",
      "(10, 1)\n",
      "(14, 831)\n",
      "(1, 1)\n",
      "(14, 831)\n",
      "(86, 1)\n",
      "[[9.89790597e-01 1.02094029e-02]\n",
      " [9.84020074e-01 1.59799259e-02]\n",
      " [5.97738061e-01 4.02261939e-01]\n",
      " [7.83178981e-01 2.16821019e-01]\n",
      " [9.45450780e-01 5.45492197e-02]\n",
      " [7.11402224e-03 9.92885978e-01]\n",
      " [9.90294559e-01 9.70544104e-03]\n",
      " [9.98846080e-01 1.15392026e-03]\n",
      " [9.99743471e-01 2.56529184e-04]\n",
      " [9.88422485e-01 1.15775152e-02]\n",
      " [8.16541936e-01 1.83458064e-01]\n",
      " [9.98822266e-01 1.17773368e-03]\n",
      " [8.51667806e-01 1.48332194e-01]\n",
      " [3.87901783e-02 9.61209822e-01]]\n",
      "(14, 831)\n",
      "(1, 1)\n",
      "[[0.68965348 0.31034652]\n",
      " [0.94472713 0.05527287]\n",
      " [0.97198464 0.02801536]\n",
      " [0.52630151 0.47369849]\n",
      " [0.83315857 0.16684143]\n",
      " [0.01922507 0.98077493]\n",
      " [0.9202656  0.0797344 ]\n",
      " [0.97060779 0.02939221]\n",
      " [0.96321432 0.03678568]\n",
      " [0.94297039 0.05702961]\n",
      " [0.75207424 0.24792576]\n",
      " [0.9763693  0.0236307 ]\n",
      " [0.98001396 0.01998604]\n",
      " [0.95341011 0.04658989]]\n",
      "(14, 831)\n",
      "(2, 1)\n",
      "(14, 831)\n",
      "(2, 1)\n",
      "(14, 831)\n",
      "(86, 1)\n",
      "(14, 831)\n",
      "(44, 1)\n",
      "[[0.0780129  0.9219871 ]\n",
      " [0.81249627 0.18750373]\n",
      " [0.9891982  0.0108018 ]\n",
      " [0.62601668 0.37398332]\n",
      " [0.05380829 0.94619171]\n",
      " [0.09748703 0.90251297]\n",
      " [0.54223211 0.45776789]\n",
      " [0.01614793 0.98385207]\n",
      " [0.32552017 0.67447983]\n",
      " [0.34643696 0.65356304]\n",
      " [0.40832787 0.59167213]\n",
      " [0.053865   0.946135  ]\n",
      " [0.99485723 0.00514277]\n",
      " [0.97883054 0.02116946]]\n",
      "(14, 831)\n",
      "(1, 1)\n",
      "(14, 831)\n",
      "(5, 1)\n",
      "(14, 831)\n",
      "(1, 1)\n",
      "(14, 831)\n",
      "(44, 1)\n"
     ]
    }
   ],
   "source": [
    "X_train = pd.read_csv('X_train.csv')\n",
    "X_test = pd.read_csv('X_test.csv')\n",
    "Y_train = pd.read_csv('Y_train.csv')\n",
    "Y_test = pd.read_csv('Y_test.csv')\n",
    "weight_matrix = pd.read_csv('weight_matrix.csv', header = None).values\n",
    "prob_matrix = []\n",
    "\n",
    "for row_index in range(5):\n",
    "    for col_index in range(5):\n",
    "        if row_index != col_index:\n",
    "            model_name = 'model_' + str(row_index + 1) + '_' + str(col_index + 1) + '.pkl'\n",
    "            clf = joblib.load(model_name) \n",
    "            col_kept_csv = 'col_kept_' + str(row_index + 1) + '_' + str(col_index + 1) + '_df.csv'\n",
    "            col_kept = pd.read_csv(col_kept_csv, header = None)\n",
    "            \n",
    "            X_test_filtered_unlabeled = StandardScaler().fit_transform(X_test)\n",
    "            X_test_filtered_unlabeled = pd.DataFrame(data = X_test_filtered_unlabeled)\n",
    "            print(X_test_filtered_unlabeled.shape)\n",
    "            print(col_kept.shape)\n",
    "            \n",
    "            X_test_FS = X_test_filtered_unlabeled[X_test_filtered_unlabeled.columns[np.transpose(col_kept.values)[0]]]\n",
    "            if (row_index < col_index):\n",
    "                print(clf.predict_proba(X_test_FS))\n",
    "                prob_matrix.append(clf.predict_proba(X_test_FS)[:, 0])\n",
    "            else:\n",
    "                prob_matrix.append(clf.predict_proba(X_test_FS)[:, 1])\n",
    "\n",
    "prob_matrix = np.array(prob_matrix).reshape(5, 4, 14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aggregate the classification result with Learning Valued Preference for Classification (LVPC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Learning Valued Preference for Classification (LVPC) E. Hüllermeier and K. Brinker. Learning valued preference structures for solving classification problems. Fuzzy Sets and Systems, 159(18):2337–2352, 2008 AND J.C. Huhn and E. Hüllermeier. FR3: A fuzzy rule learner for inducing reliable classifiers. IEEE Transactions on Fuzzy Systems, 17(1):138–149, 2009. This method consider the score matrix as a fuzzy preference relation, based in fuzzy preference modeling the original relation is decomposed in three new relations with different meanings, the strict preference, the conflict and the ignorance. A decision rule based on voting strategy is proposed to obtain the output class from them (https://sci2s.ugr.es/ovo-ova):"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation*}\n",
    "Class = argmax_{i = 1,...,m} \\sum_{1\\leq j \\neq i \\leq m} P_{ijk} + \\frac{1}{2}C_{ijk} + \\frac{N_{i}}{N_{i} + N_{j}} I_{ijk}\n",
    "\\end{equation*}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "where:\n",
    "- N<sub>i</sub> is the number of examples from class i in the training data (an unbiased estimate of the class -- probability)\n",
    "- C<sub>ijk</sub> is the degree of conflict (the degree to which both classes are supported)\n",
    "- I<sub>ijk</sub> is the degree of ignorance (the degree to which none of the classes is supported) and finally,\n",
    "- P<sub>ijk</sub> and P<sub>jik</sub> are respectively the strict preference for i and j. Preference, confidence and ignorance degrees are computed as follows:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{align}\n",
    "P_{ijk} = r_{ijk} - min(r_{ijk}, r_{jik}) \\\\\n",
    "P_{jik} = r_{jik} - min(r_{ijk}, r_{jik}) \\\\\n",
    "C_{ijk} = min(r_{ijk}, r_{jik}) \\\\\n",
    "I_{ijk} = 1 - min(r_{ijk}, r_{jik}) \\\\\n",
    "\\end{align}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P:\n",
      "[[[0.98016105 0.39223417 0.99955838 0.99997707 0.53753009 0.\n",
      "   0.         0.         0.66113251 0.97189199 0.99177646 0.\n",
      "   0.99981046 0.9993928 ]\n",
      "  [0.78441139 0.         0.87906327 0.99965841 0.         0.\n",
      "   0.         0.         0.10347637 0.41685297 0.49968118 0.\n",
      "   0.99751465 0.98173371]\n",
      "  [0.81084581 0.85873908 0.84348021 0.99956716 0.0728065  0.\n",
      "   0.02956777 0.71461514 0.72238959 0.9000969  0.878211   0.99929004\n",
      "   0.94728314 0.89884607]\n",
      "  [0.94103177 0.90642465 0.94103177 0.94103177 0.         0.\n",
      "   0.06630025 0.90109626 0.77439437 0.76708027 0.94103177 0.99995194\n",
      "   0.94103177 0.94103177]]\n",
      "\n",
      " [[0.         0.         0.         0.         0.         0.41785568\n",
      "   0.17256174 0.87059809 0.         0.         0.         0.90080252\n",
      "   0.         0.        ]\n",
      "  [0.         0.         0.         0.         0.         0.38216946\n",
      "   0.         0.0950001  0.         0.         0.         0.\n",
      "   0.         0.        ]\n",
      "  [0.09790159 0.         0.61313754 0.85661878 0.         0.\n",
      "   0.         0.         0.20643855 0.10740149 0.95603423 0.\n",
      "   0.61313754 0.61313754]\n",
      "  [0.         0.78053247 0.25909764 0.47963779 0.         0.\n",
      "   0.         0.91450671 0.76575252 0.88279397 0.7465747  0.84036959\n",
      "   0.64697489 0.29045147]]\n",
      "\n",
      " [[0.         0.88846305 0.         0.         0.99956179 0.95057445\n",
      "   0.97950323 0.99975931 0.         0.         0.         0.96626204\n",
      "   0.         0.        ]\n",
      "  [0.54830076 0.48040761 0.55392279 0.55395958 0.52161931 0.\n",
      "   0.52646426 0.         0.5519862  0.55300555 0.54822008 0.26503467\n",
      "   0.55370836 0.55396009]\n",
      "  [0.97958119 0.96804015 0.19547612 0.56635796 0.89090156 0.\n",
      "   0.98058912 0.99769216 0.99948694 0.97684497 0.63308387 0.99764453\n",
      "   0.70333561 0.        ]\n",
      "  [0.37930696 0.88945425 0.94396928 0.05260303 0.66631714 0.\n",
      "   0.8405312  0.94121559 0.92642864 0.88594077 0.50414849 0.95273861\n",
      "   0.96002792 0.90682021]]\n",
      "\n",
      " [[0.         0.         0.         0.         0.         0.61621443\n",
      "   0.         0.         0.         0.         0.         0.\n",
      "   0.         0.        ]\n",
      "  [0.         0.31446504 0.         0.         0.52566439 0.87340206\n",
      "   0.8672733  0.51335808 0.         0.         0.         0.33693854\n",
      "   0.         0.        ]\n",
      "  [0.         0.         0.         0.         0.         0.98577196\n",
      "   0.         0.         0.         0.         0.         0.\n",
      "   0.         0.92241964]\n",
      "  [0.         0.62499254 0.97839641 0.25203336 0.         0.\n",
      "   0.08446423 0.         0.         0.         0.         0.\n",
      "   0.98971447 0.95766108]]\n",
      "\n",
      " [[0.         0.         0.         0.         0.23996723 0.38873884\n",
      "   0.         0.         0.         0.         0.         0.\n",
      "   0.         0.        ]\n",
      "  [0.88336848 0.         0.         0.         0.91620396 0.95505509\n",
      "   0.2082462  0.         0.         0.         0.         0.\n",
      "   0.         0.        ]\n",
      "  [0.         0.         0.         0.         0.         0.96154987\n",
      "   0.         0.         0.         0.         0.         0.\n",
      "   0.         0.        ]\n",
      "  [0.84397419 0.         0.         0.         0.89238342 0.80502594\n",
      "   0.         0.96770415 0.34895966 0.30712609 0.18334426 0.89227001\n",
      "   0.         0.        ]]]\n",
      "C:\n",
      "[[[9.91947617e-03 3.03882915e-01 2.20808277e-04 1.14649837e-05\n",
      "   2.31234956e-01 2.91072158e-01 4.13719129e-01 6.47009545e-02\n",
      "   1.69433745e-01 1.40540070e-02 4.11176969e-03 4.95987422e-02\n",
      "   9.47683658e-05 3.03602065e-04]\n",
      "  [1.07794306e-01 5.57684775e-02 6.04683658e-02 1.70796712e-04\n",
      "   2.19103839e-04 2.47127750e-02 1.02483835e-02 1.20346224e-04\n",
      "   4.48261814e-01 2.91573514e-01 2.50159409e-01 1.68689806e-02\n",
      "   1.24267744e-03 9.13314655e-03]\n",
      "  [9.45770934e-02 7.06304612e-02 7.82598937e-02 2.16420922e-04\n",
      "   4.63596749e-01 1.91892786e-01 4.85216113e-01 1.42692430e-01\n",
      "   1.38805203e-01 4.99515511e-02 6.08944988e-02 3.54978837e-04\n",
      "   2.63584295e-02 5.05769645e-02]\n",
      "  [2.94841134e-02 4.67876750e-02 2.94841134e-02 2.94841134e-02\n",
      "   3.80016387e-01 3.05630580e-01 4.66849873e-01 4.94518713e-02\n",
      "   1.12802814e-01 1.16459864e-01 2.94841134e-02 2.40295886e-05\n",
      "   2.94841134e-02 2.94841134e-02]]\n",
      "\n",
      " [[9.91947617e-03 3.03882915e-01 2.20808277e-04 1.14649837e-05\n",
      "   2.31234956e-01 2.91072158e-01 4.13719129e-01 6.47009545e-02\n",
      "   1.69433745e-01 1.40540070e-02 4.11176969e-03 4.95987422e-02\n",
      "   9.47683658e-05 3.03602065e-04]\n",
      "  [2.25849621e-01 2.59796193e-01 2.23038606e-01 2.23020209e-01\n",
      "   2.39190346e-01 3.08915268e-01 2.36767871e-01 4.52499950e-01\n",
      "   2.24006902e-01 2.23497226e-01 2.25889959e-01 3.67482667e-01\n",
      "   2.23145818e-01 2.23019957e-01]\n",
      "  [4.51049207e-01 3.42767481e-01 1.93431232e-01 7.16906097e-02\n",
      "   2.37167806e-01 6.32989702e-02 6.63633514e-02 2.43320960e-01\n",
      "   3.96780726e-01 4.46299253e-01 2.19828831e-02 3.31530728e-01\n",
      "   1.93431232e-01 1.93431232e-01]\n",
      "  [5.83157601e-02 1.09733764e-01 3.70451181e-01 2.60181105e-01\n",
      "   4.18980192e-02 2.24724545e-02 3.95876901e-01 4.27466462e-02\n",
      "   1.17123738e-01 5.86030145e-02 1.26712650e-01 7.98152030e-02\n",
      "   1.76512556e-01 3.54774265e-01]]\n",
      "\n",
      " [[1.07794306e-01 5.57684775e-02 6.04683658e-02 1.70796712e-04\n",
      "   2.19103839e-04 2.47127750e-02 1.02483835e-02 1.20346224e-04\n",
      "   4.48261814e-01 2.91573514e-01 2.50159409e-01 1.68689806e-02\n",
      "   1.24267744e-03 9.13314655e-03]\n",
      "  [2.25849621e-01 2.59796193e-01 2.23038606e-01 2.23020209e-01\n",
      "   2.39190346e-01 3.08915268e-01 2.36767871e-01 4.52499950e-01\n",
      "   2.24006902e-01 2.23497226e-01 2.25889959e-01 3.67482667e-01\n",
      "   2.23145818e-01 2.23019957e-01]\n",
      "  [1.02094029e-02 1.59799259e-02 4.02261939e-01 2.16821019e-01\n",
      "   5.45492197e-02 7.11402224e-03 9.70544104e-03 1.15392026e-03\n",
      "   2.56529184e-04 1.15775152e-02 1.83458064e-01 1.17773368e-03\n",
      "   1.48332194e-01 3.87901783e-02]\n",
      "  [3.10346521e-01 5.52728741e-02 2.80153599e-02 4.73698486e-01\n",
      "   1.66841428e-01 1.92250662e-02 7.97344003e-02 2.93922074e-02\n",
      "   3.67856816e-02 5.70296149e-02 2.47925756e-01 2.36306954e-02\n",
      "   1.99860408e-02 4.65898930e-02]]\n",
      "\n",
      " [[9.45770934e-02 7.06304612e-02 7.82598937e-02 2.16420922e-04\n",
      "   4.63596749e-01 1.91892786e-01 4.85216113e-01 1.42692430e-01\n",
      "   1.38805203e-01 4.99515511e-02 6.08944988e-02 3.54978837e-04\n",
      "   2.63584295e-02 5.05769645e-02]\n",
      "  [4.51049207e-01 3.42767481e-01 1.93431232e-01 7.16906097e-02\n",
      "   2.37167806e-01 6.32989702e-02 6.63633514e-02 2.43320960e-01\n",
      "   3.96780726e-01 4.46299253e-01 2.19828831e-02 3.31530728e-01\n",
      "   1.93431232e-01 1.93431232e-01]\n",
      "  [1.02094029e-02 1.59799259e-02 4.02261939e-01 2.16821019e-01\n",
      "   5.45492197e-02 7.11402224e-03 9.70544104e-03 1.15392026e-03\n",
      "   2.56529184e-04 1.15775152e-02 1.83458064e-01 1.17773368e-03\n",
      "   1.48332194e-01 3.87901783e-02]\n",
      "  [7.80129032e-02 1.87503728e-01 1.08017958e-02 3.73983318e-01\n",
      "   5.38082883e-02 9.74870278e-02 4.57767885e-01 1.61479273e-02\n",
      "   3.25520170e-01 3.46436956e-01 4.08327870e-01 5.38649963e-02\n",
      "   5.14276658e-03 2.11694616e-02]]\n",
      "\n",
      " [[2.94841134e-02 4.67876750e-02 2.94841134e-02 2.94841134e-02\n",
      "   3.80016387e-01 3.05630580e-01 4.66849873e-01 4.94518713e-02\n",
      "   1.12802814e-01 1.16459864e-01 2.94841134e-02 2.40295886e-05\n",
      "   2.94841134e-02 2.94841134e-02]\n",
      "  [5.83157601e-02 1.09733764e-01 3.70451181e-01 2.60181105e-01\n",
      "   4.18980192e-02 2.24724545e-02 3.95876901e-01 4.27466462e-02\n",
      "   1.17123738e-01 5.86030145e-02 1.26712650e-01 7.98152030e-02\n",
      "   1.76512556e-01 3.54774265e-01]\n",
      "  [3.10346521e-01 5.52728741e-02 2.80153599e-02 4.73698486e-01\n",
      "   1.66841428e-01 1.92250662e-02 7.97344003e-02 2.93922074e-02\n",
      "   3.67856816e-02 5.70296149e-02 2.47925756e-01 2.36306954e-02\n",
      "   1.99860408e-02 4.65898930e-02]\n",
      "  [7.80129032e-02 1.87503728e-01 1.08017958e-02 3.73983318e-01\n",
      "   5.38082883e-02 9.74870278e-02 4.57767885e-01 1.61479273e-02\n",
      "   3.25520170e-01 3.46436956e-01 4.08327870e-01 5.38649963e-02\n",
      "   5.14276658e-03 2.11694616e-02]]]\n",
      "I:\n",
      "[[[9.91947617e-03 3.03882915e-01 2.20808277e-04 1.14649837e-05\n",
      "   2.31234956e-01 2.91072158e-01 4.13719129e-01 6.47009545e-02\n",
      "   1.69433745e-01 1.40540070e-02 4.11176969e-03 4.95987422e-02\n",
      "   9.47683658e-05 3.03602065e-04]\n",
      "  [1.07794306e-01 5.57684775e-02 6.04683658e-02 1.70796712e-04\n",
      "   2.19103839e-04 2.47127750e-02 1.02483835e-02 1.20346224e-04\n",
      "   4.48261814e-01 2.91573514e-01 2.50159409e-01 1.68689806e-02\n",
      "   1.24267744e-03 9.13314655e-03]\n",
      "  [9.45770934e-02 7.06304612e-02 7.82598937e-02 2.16420922e-04\n",
      "   4.63596749e-01 1.91892786e-01 4.85216113e-01 1.42692430e-01\n",
      "   1.38805203e-01 4.99515511e-02 6.08944988e-02 3.54978837e-04\n",
      "   2.63584295e-02 5.05769645e-02]\n",
      "  [2.94841134e-02 4.67876750e-02 2.94841134e-02 2.94841134e-02\n",
      "   3.80016387e-01 3.05630580e-01 4.66849873e-01 4.94518713e-02\n",
      "   1.12802814e-01 1.16459864e-01 2.94841134e-02 2.40295886e-05\n",
      "   2.94841134e-02 2.94841134e-02]]\n",
      "\n",
      " [[9.91947617e-03 3.03882915e-01 2.20808277e-04 1.14649837e-05\n",
      "   2.31234956e-01 2.91072158e-01 4.13719129e-01 6.47009545e-02\n",
      "   1.69433745e-01 1.40540070e-02 4.11176969e-03 4.95987422e-02\n",
      "   9.47683658e-05 3.03602065e-04]\n",
      "  [2.25849621e-01 2.59796193e-01 2.23038606e-01 2.23020209e-01\n",
      "   2.39190346e-01 3.08915268e-01 2.36767871e-01 4.52499950e-01\n",
      "   2.24006902e-01 2.23497226e-01 2.25889959e-01 3.67482667e-01\n",
      "   2.23145818e-01 2.23019957e-01]\n",
      "  [4.51049207e-01 3.42767481e-01 1.93431232e-01 7.16906097e-02\n",
      "   2.37167806e-01 6.32989702e-02 6.63633514e-02 2.43320960e-01\n",
      "   3.96780726e-01 4.46299253e-01 2.19828831e-02 3.31530728e-01\n",
      "   1.93431232e-01 1.93431232e-01]\n",
      "  [5.83157601e-02 1.09733764e-01 3.70451181e-01 2.60181105e-01\n",
      "   4.18980192e-02 2.24724545e-02 3.95876901e-01 4.27466462e-02\n",
      "   1.17123738e-01 5.86030145e-02 1.26712650e-01 7.98152030e-02\n",
      "   1.76512556e-01 3.54774265e-01]]\n",
      "\n",
      " [[1.07794306e-01 5.57684775e-02 6.04683658e-02 1.70796712e-04\n",
      "   2.19103839e-04 2.47127750e-02 1.02483835e-02 1.20346224e-04\n",
      "   4.48261814e-01 2.91573514e-01 2.50159409e-01 1.68689806e-02\n",
      "   1.24267744e-03 9.13314655e-03]\n",
      "  [2.25849621e-01 2.59796193e-01 2.23038606e-01 2.23020209e-01\n",
      "   2.39190346e-01 3.08915268e-01 2.36767871e-01 4.52499950e-01\n",
      "   2.24006902e-01 2.23497226e-01 2.25889959e-01 3.67482667e-01\n",
      "   2.23145818e-01 2.23019957e-01]\n",
      "  [1.02094029e-02 1.59799259e-02 4.02261939e-01 2.16821019e-01\n",
      "   5.45492197e-02 7.11402224e-03 9.70544104e-03 1.15392026e-03\n",
      "   2.56529184e-04 1.15775152e-02 1.83458064e-01 1.17773368e-03\n",
      "   1.48332194e-01 3.87901783e-02]\n",
      "  [3.10346521e-01 5.52728741e-02 2.80153599e-02 4.73698486e-01\n",
      "   1.66841428e-01 1.92250662e-02 7.97344003e-02 2.93922074e-02\n",
      "   3.67856816e-02 5.70296149e-02 2.47925756e-01 2.36306954e-02\n",
      "   1.99860408e-02 4.65898930e-02]]\n",
      "\n",
      " [[9.45770934e-02 7.06304612e-02 7.82598937e-02 2.16420922e-04\n",
      "   4.63596749e-01 1.91892786e-01 4.85216113e-01 1.42692430e-01\n",
      "   1.38805203e-01 4.99515511e-02 6.08944988e-02 3.54978837e-04\n",
      "   2.63584295e-02 5.05769645e-02]\n",
      "  [4.51049207e-01 3.42767481e-01 1.93431232e-01 7.16906097e-02\n",
      "   2.37167806e-01 6.32989702e-02 6.63633514e-02 2.43320960e-01\n",
      "   3.96780726e-01 4.46299253e-01 2.19828831e-02 3.31530728e-01\n",
      "   1.93431232e-01 1.93431232e-01]\n",
      "  [1.02094029e-02 1.59799259e-02 4.02261939e-01 2.16821019e-01\n",
      "   5.45492197e-02 7.11402224e-03 9.70544104e-03 1.15392026e-03\n",
      "   2.56529184e-04 1.15775152e-02 1.83458064e-01 1.17773368e-03\n",
      "   1.48332194e-01 3.87901783e-02]\n",
      "  [7.80129032e-02 1.87503728e-01 1.08017958e-02 3.73983318e-01\n",
      "   5.38082883e-02 9.74870278e-02 4.57767885e-01 1.61479273e-02\n",
      "   3.25520170e-01 3.46436956e-01 4.08327870e-01 5.38649963e-02\n",
      "   5.14276658e-03 2.11694616e-02]]\n",
      "\n",
      " [[2.94841134e-02 4.67876750e-02 2.94841134e-02 2.94841134e-02\n",
      "   3.80016387e-01 3.05630580e-01 4.66849873e-01 4.94518713e-02\n",
      "   1.12802814e-01 1.16459864e-01 2.94841134e-02 2.40295886e-05\n",
      "   2.94841134e-02 2.94841134e-02]\n",
      "  [5.83157601e-02 1.09733764e-01 3.70451181e-01 2.60181105e-01\n",
      "   4.18980192e-02 2.24724545e-02 3.95876901e-01 4.27466462e-02\n",
      "   1.17123738e-01 5.86030145e-02 1.26712650e-01 7.98152030e-02\n",
      "   1.76512556e-01 3.54774265e-01]\n",
      "  [3.10346521e-01 5.52728741e-02 2.80153599e-02 4.73698486e-01\n",
      "   1.66841428e-01 1.92250662e-02 7.97344003e-02 2.93922074e-02\n",
      "   3.67856816e-02 5.70296149e-02 2.47925756e-01 2.36306954e-02\n",
      "   1.99860408e-02 4.65898930e-02]\n",
      "  [7.80129032e-02 1.87503728e-01 1.08017958e-02 3.73983318e-01\n",
      "   5.38082883e-02 9.74870278e-02 4.57767885e-01 1.61479273e-02\n",
      "   3.25520170e-01 3.46436956e-01 4.08327870e-01 5.38649963e-02\n",
      "   5.14276658e-03 2.11694616e-02]]]\n"
     ]
    }
   ],
   "source": [
    "minimum_r = []\n",
    "maximum_r = []\n",
    "# Create a list of class labels\n",
    "list = [1, 2, 3, 4, 5]\n",
    "# Create a list of tuples for binary classifier. e.g. (1, 2)\n",
    "class_pair_combo_2 = [(x, y) for x in list for y in list if x != y]\n",
    "for pair in class_pair_combo_2:\n",
    "    if pair[0] < pair[1]:\n",
    "        small_class = pair[0]\n",
    "        large_class = pair[1]\n",
    "    else:\n",
    "        small_class = pair[1]\n",
    "        large_class = pair[0]\n",
    "    temp_min = np.minimum(prob_matrix[small_class - 1][large_class - 2], prob_matrix[large_class - 1][small_class - 1])\n",
    "    temp_max = np.maximum(prob_matrix[small_class - 1][large_class - 2], prob_matrix[large_class - 1][small_class - 1])\n",
    "    minimum_r.append(temp_min)\n",
    "    maximum_r.append(temp_max)\n",
    "    \n",
    "minimum_r = np.array(minimum_r).reshape(5, 4, 14)\n",
    "maximum_r = np.array(maximum_r).reshape(5, 4, 14)\n",
    "\n",
    "# Calculating P, C, and I\n",
    "P = np.subtract(prob_matrix, minimum_r)\n",
    "C = minimum_r\n",
    "I = np.ones((5, 4, 14), dtype = float) - maximum_r\n",
    "print('P:')\n",
    "print(P)\n",
    "print('C:')\n",
    "print(C)\n",
    "print('I:')\n",
    "print(I)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_size_matrix = [[49, 49, 49, 49],\n",
    "                     [16, 16, 16, 16],\n",
    "                     [45, 45, 45, 45],\n",
    "                     [14, 14, 14, 14],\n",
    "                     [12, 12, 12, 12]]\n",
    "# Calculate the proportion size matrix later for LVPC\n",
    "proportion_size_matrix = np.divide(class_size_matrix, weight_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7.4125617  1.41375277 5.00705465 1.0699921  4.71677139]\n",
      " [5.24103877 3.3630873  7.01964278 3.30441313 0.66827075]\n",
      " [7.58554446 3.34765887 4.69303949 3.30301544 0.77264241]\n",
      " [7.92167011 4.00872309 3.92056008 1.71716557 1.76603608]\n",
      " [2.96605417 1.04172592 6.79970212 2.30448345 5.44242455]\n",
      " [1.2832444  2.15957035 2.16791648 5.1599324  6.96927637]\n",
      " [2.38021799 1.90151534 7.13026951 3.73564511 2.68726404]\n",
      " [3.79515521 4.90859826 6.55310082 1.79357721 2.47675838]\n",
      " [5.90694723 3.7154132  6.16681494 1.48911918 1.8111035 ]\n",
      " [6.87570659 3.57527157 5.89877919 1.53160137 1.67082636]\n",
      " [7.19431313 4.54591079 4.8943981  1.11103588 1.74812266]\n",
      " [4.28724552 4.7297134  6.94881252 1.46247244 2.37592716]\n",
      " [7.8519773  3.91604428 5.299018   2.96329438 0.39316918]\n",
      " [7.7715917  3.39367572 3.60109626 4.35964261 0.78466068]]\n",
      "[[0.00367925 0.00109213 0.00239699 0.00076615 0.00265268]\n",
      " [0.00234568 0.0023516  0.0037988  0.00162323 0.00053144]\n",
      " [0.00376353 0.00186483 0.0022142  0.00208228 0.00052253]\n",
      " [0.00391145 0.00209358 0.00190854 0.00108424 0.00171324]\n",
      " [0.0014976  0.00125583 0.00376519 0.00195138 0.00374812]\n",
      " [0.00072653 0.00297738 0.0017272  0.00476495 0.0055871 ]\n",
      " [0.00130242 0.00216791 0.00387072 0.00254144 0.00233968]\n",
      " [0.00163939 0.00423889 0.00364913 0.00111273 0.00124264]\n",
      " [0.00298705 0.00225541 0.00314211 0.00101547 0.00115852]\n",
      " [0.00340507 0.00192218 0.00288668 0.00091717 0.00112637]\n",
      " [0.0035316  0.00228388 0.00249771 0.00089526 0.00133101]\n",
      " [0.00181746 0.0039741  0.00379684 0.00074906 0.00111682]\n",
      " [0.00388649 0.00204504 0.00237068 0.00147523 0.00029812]\n",
      " [0.00385438 0.00187954 0.00160474 0.00311438 0.00055566]]\n"
     ]
    }
   ],
   "source": [
    "weighted_prob = []\n",
    "class_score = []\n",
    "for c in range(5):\n",
    "    weighted_prob.append(np.average(prob_matrix[c], axis = 0, weights = weight_matrix[c]) / np.sum(weight_matrix[c]))\n",
    "    class_score.append((np.average(P[c], axis = 0) + 0.5 * np.average(C[c], axis = 0) + np.average(prob_matrix[c], axis = 0, weights = proportion_size_matrix[c])) * 4)\n",
    "weighted_prob = np.transpose(np.array(weighted_prob))\n",
    "class_score = np.transpose(np.array(class_score))\n",
    "print(class_score)\n",
    "print(weighted_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.18028337 0.01747405 0.10786456 0.01072605 0.03183221]\n",
      " [0.11493816 0.03762555 0.17094614 0.02272523 0.00637731]\n",
      " [0.18441276 0.0298373  0.09963889 0.02915191 0.00627033]\n",
      " [0.19166095 0.03349725 0.08588426 0.01517941 0.02055892]\n",
      " [0.07338216 0.02009332 0.16943365 0.02731927 0.0449774 ]\n",
      " [0.03559997 0.04763801 0.0777242  0.06670924 0.0670452 ]\n",
      " [0.06381867 0.03468655 0.17418254 0.03558015 0.02807612]\n",
      " [0.08032992 0.06782223 0.16421068 0.01557824 0.01491168]\n",
      " [0.14636525 0.0360866  0.14139497 0.01421662 0.01390228]\n",
      " [0.16684865 0.03075493 0.12990069 0.01284033 0.01351638]\n",
      " [0.17304846 0.03654213 0.1123968  0.01253362 0.01597213]\n",
      " [0.08905531 0.06358556 0.17085783 0.01048687 0.01340185]\n",
      " [0.19043813 0.0327206  0.10668052 0.02065328 0.00357743]\n",
      " [0.18886467 0.03007272 0.07221332 0.04360135 0.00666791]]\n"
     ]
    }
   ],
   "source": [
    "class_size = [49, 16, 45, 14, 12]\n",
    "for col in range(5):\n",
    "    weighted_prob[:,col] *= class_size[col]\n",
    "print(weighted_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_class = []\n",
    "for r in range(len(weighted_prob)):\n",
    "    predicted_class.append(np.where(class_score[r] == np.amax(class_score[r]))[0][0] + 1)\n",
    "predicted_class = np.transpose(predicted_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5714285714285714\n"
     ]
    }
   ],
   "source": [
    "num_correct = 0\n",
    "for index in range(len(predicted_class)):\n",
    "    if predicted_class[index] == np.transpose(Y_test.values)[0][index]:\n",
    "        num_correct += 1\n",
    "print(num_correct/len(predicted_class))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}