FML_Project_PreprocessingData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5e0b6148",
   "metadata": {},
   "source": [
    "## Step 1: Preprocess data <br>\n",
    "For consistency reasons and to be able to fuse our work together we should use the same test/training data for our different models\n",
    "\n",
    "\n",
    "a) Import <br>\n",
    "Labels are in a seperate file. Confusing folder structure\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "405a991e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NaN values: \n",
      " Path (Trial/Arm/Folder/Visit/Eye/Image Name)    0\n",
      "Scan (n/49)                                     0\n",
      "Atrophy / thinning of retinal layers            0\n",
      "Disruption of EZ                                0\n",
      "DRIL                                            0\n",
      "IR hemorrhages                                  0\n",
      "IR HRF                                          0\n",
      "Partially attached vitreous face                6\n",
      "Fully attached vitreous face                    3\n",
      "Preretinal tissue/hemorrhage                    2\n",
      "Vitreous debris                                 8\n",
      "VMT                                             0\n",
      "DRT/ME                                          0\n",
      "Fluid (IRF)                                     0\n",
      "Fluid (SRF)                                     0\n",
      "Disruption of RPE                               0\n",
      "PED (serous)                                    0\n",
      "SHRM                                            0\n",
      "Eye_ID                                          0\n",
      "BCVA                                            0\n",
      "CST                                             0\n",
      "Patient_ID                                      0\n",
      "dtype: int64 \n",
      "\n",
      "Present percentages for each biomarker: \n",
      " Atrophy / thinning of retinal layers     1.764\n",
      "Disruption of EZ                         6.420\n",
      "DRIL                                     0.340\n",
      "IR hemorrhages                           3.965\n",
      "IR HRF                                  67.400\n",
      "Partially attached vitreous face        31.718\n",
      "Fully attached vitreous face            55.506\n",
      "Preretinal tissue/hemorrhage             8.578\n",
      "Vitreous debris                         30.145\n",
      "VMT                                      0.106\n",
      "DRT/ME                                  31.920\n",
      "Fluid (IRF)                             43.452\n",
      "Fluid (SRF)                              2.477\n",
      "Disruption of RPE                        0.106\n",
      "PED (serous)                             0.106\n",
      "SHRM                                     0.808\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# loading the label file (DataFrame)\n",
    "BiomarkerLabel_df = pd.read_csv('OLIVES_Dataset_Labels/Biomarker_Clinical_Data_Images.csv')\n",
    "\n",
    "# There is a mistake in the initial file path we have to fix\n",
    "# /TREX DME/... should be /TREX_DME/...\n",
    "BiomarkerLabel_df['Path (Trial/Arm/Folder/Visit/Eye/Image Name)'] = BiomarkerLabel_df['Path (Trial/Arm/Folder/Visit/Eye/Image Name)'].str.replace('/TREX DME/', '/TREX_DME/', regex=False)\n",
    "\n",
    "# Also some cells are corrupted and show a NaN\n",
    "# This will show how many NaN values are in each column\n",
    "print(\"NaN values: \\n\", BiomarkerLabel_df.isna().sum(), \"\\n\")\n",
    "# Since these are not a lot we will just replace them with a 0\n",
    "BiomarkerLabel_df = BiomarkerLabel_df.fillna(0)\n",
    "\n",
    "# All biomarker and clinical data should be integers but they aren't so we have to clean it up\n",
    "feature_columns = BiomarkerLabel_df.columns[2:22]\n",
    "BiomarkerLabel_df[feature_columns] = BiomarkerLabel_df[feature_columns].astype(int)\n",
    "\n",
    "## Extract data\n",
    "# Get image paths as your features\n",
    "df_features = BiomarkerLabel_df['Path (Trial/Arm/Folder/Visit/Eye/Image Name)']\n",
    "# Get scan number\n",
    "df_scan = BiomarkerLabel_df['Scan (n/49)']\n",
    "# Get the biomarker data (target)\n",
    "labels = BiomarkerLabel_df.columns[2:18] # Extract label columns names (biomarkers)\n",
    "df_labels = BiomarkerLabel_df[labels] \n",
    "# Get clinical data columns\n",
    "df_clinicalData = BiomarkerLabel_df[['Eye_ID', 'BCVA', 'CST', 'Patient_ID']]\n",
    "\n",
    "# Calculate the percentages of zero values for each biomarker\n",
    "print(\"Present percentages for each biomarker: \\n\", ((df_labels == 1).sum() / len(df_labels) * 100).round(3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "defea404",
   "metadata": {},
   "source": [
    "b) Train/Test split <br>\n",
    "We want an equal split. Also split the clinical data together.\n",
    "\n",
    "There are a lot of considerations that can be made here:\n",
    "- Do we want to keep sets of scans for one eye together?\n",
    "- What do we use to stratify? All biomarkers together create too many different combinations of biomarkers (one combination only exists once -> error)\n",
    "    - Use the oversampled biomarkers? (Uncommon split won't be controlled, however we don't expect good results for these anyways)\n",
    "    - Use the undersampled ones? (And let the common ones just be randomly splitted?) Create artificial data points to multiple the appearence of these \n",
    "- Do we make a difference for the two initial datasets which were put together here?\n",
    "\n",
    "<br>\n",
    "How do we manage the Inbalance?\n",
    "\n",
    "- Overrepresented Classes (Large sample sizes):\n",
    "    - [5] IR HRF (6341)\n",
    "    - [6] Fully attached vitreous face (5222)\n",
    "    - [11] Fluid (IRF) (4088)\n",
    "    - [10] DRT/ME (3003)\n",
    "    - [5] Partially attached vitreous face (2984)\n",
    "    - [8] Vitreous debris (2836)\n",
    "- Moderately Represented Classes:\n",
    "    - [7] Preretinal tissue/hemorrhage (807)\n",
    "    - [1] Disruption of EZ (604)\n",
    "    - [3] IR hemorrhages (373)\n",
    "- Rare Classes:\n",
    "    - [12] Fluid (SRF) (233)\n",
    "    - [0] Atrophy / thinning of retinal layers (166)\n",
    "- Underrepresented Classes:\n",
    "    - [15] SHRM (76)\n",
    "    - [2] DRIL (32)\n",
    "    - [9] VMT (10)\n",
    "    - [13] Disruption of RPE (10)\n",
    "    - [14] PED (serous) (10)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a98e097d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label counts: \n",
      " Atrophy / thinning of retinal layers     166\n",
      "Disruption of EZ                         604\n",
      "DRIL                                      32\n",
      "IR hemorrhages                           373\n",
      "IR HRF                                  6341\n",
      "Partially attached vitreous face        2984\n",
      "Fully attached vitreous face            5222\n",
      "Preretinal tissue/hemorrhage             807\n",
      "Vitreous debris                         2836\n",
      "VMT                                       10\n",
      "DRT/ME                                  3003\n",
      "Fluid (IRF)                             4088\n",
      "Fluid (SRF)                              233\n",
      "Disruption of RPE                         10\n",
      "PED (serous)                              10\n",
      "SHRM                                      76\n",
      "dtype: int64 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Doesn't work anymore because we fixed it before\n",
    "# # This will show how many NaN values are in each biomarker column\n",
    "# print(\"NaN values: \\n\", df_labels.isna().sum(), \"\\n\")\n",
    "# # Since these are not a lot we will just replace them with a 0\n",
    "# df_labels = df_labels.fillna(0)\n",
    "\n",
    "# Check the distribution of biomarker labels (count occurrences)\n",
    "label_counts = df_labels.sum(axis=0).astype(int)\n",
    "print(\"Label counts: \\n\", label_counts, \"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3bcbde3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # OPTION 1:\n",
    "# # split by underrepresented classes\n",
    "\n",
    "# # Combine the data we want to split\n",
    "# df_combined = pd.concat([df_features, df_labels, df_clinicalData], axis=1)\n",
    "\n",
    "# # set which biomarkers will be used as stratifiers\n",
    "# # in this example the underrepresented classes will be used\n",
    "# underrepresented_classes = ['SHRM', 'DRIL', 'VMT', 'Disruption of RPE', 'PED (serous)']\n",
    "# df_labels['underrepresented'] = df_labels[underrepresented_classes].sum(axis=1) > 0\n",
    "                         \n",
    "# ## Split the data \n",
    "# # 80/20 Train/Test split\n",
    "# # We use the biomarkers data for stratisfying to make sure we get an equal distribution (doesn't quite work)\n",
    "# train_data, train_data, train_labels, test_label = train_test_split(\n",
    "#     df_combined, df_labels, \n",
    "#     test_size=0.2,\n",
    "#     random_state=0,\n",
    "#     stratify=df_labels['underrepresented']\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "797a6f21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Biomarker presence for the full eye scan (from 96 eyes):\n",
      " Atrophy / thinning of retinal layers    16\n",
      "Disruption of EZ                        46\n",
      "DRIL                                     4\n",
      "IR hemorrhages                          51\n",
      "IR HRF                                  96\n",
      "Partially attached vitreous face        67\n",
      "Fully attached vitreous face            75\n",
      "Preretinal tissue/hemorrhage            34\n",
      "Vitreous debris                         91\n",
      "VMT                                      3\n",
      "DRT/ME                                  72\n",
      "Fluid (IRF)                             93\n",
      "Fluid (SRF)                             27\n",
      "Disruption of RPE                        5\n",
      "PED (serous)                             1\n",
      "SHRM                                    14\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# OPTION 2:\n",
    "# split by biomarker presence for a complete eye. Keep eye set together for both visits.\n",
    "\n",
    "# Summarize biomarker presence for each Eye_ID group\n",
    "# If any scan in an eye group has a biomarker, the eye is marked as having the biomarker\n",
    "BiomarkerPresence_FullEye = BiomarkerLabel_df.groupby('Eye_ID')[labels].max()\n",
    "\n",
    "# Add a summarization column indicating overall biomarker presence for stratification\n",
    "BiomarkerPresence_FullEye['Biomarker Presence'] = BiomarkerPresence_FullEye.any(axis=1).astype(int)\n",
    "# Count Biomarker Presence for Each Label\n",
    "biomarker_counts = BiomarkerPresence_FullEye[labels].sum(axis=0).astype(int)\n",
    "print(\"Biomarker presence for the full eye scan (from 96 eyes):\\n\", biomarker_counts)\n",
    "\n",
    "# Save the summarized biomarker presence to a separate CSV\n",
    "BiomarkerPresence_FullEye.to_csv('OLIVES_Dataset_Labels/BiomarkerPresence_FullEye.csv', index=True, float_format='%.0f')\n",
    "\n",
    "# Stratified split based on the summarized biomarker presence\n",
    "train_eyes, test_eyes = train_test_split(\n",
    "    BiomarkerPresence_FullEye, \n",
    "    test_size=0.2, \n",
    "    random_state=0, \n",
    "    stratify=BiomarkerPresence_FullEye['Biomarker Presence']\n",
    ")\n",
    "\n",
    "# Filter the original DataFrame to include only rows corresponding to the split Eye_IDs\n",
    "train_data = BiomarkerLabel_df[BiomarkerLabel_df['Eye_ID'].isin(train_eyes.index)]\n",
    "test_data = BiomarkerLabel_df[BiomarkerLabel_df['Eye_ID'].isin(test_eyes.index)]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec004004",
   "metadata": {},
   "source": [
    "Check out the split fo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cdb0cf17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Present percentages for each biomarker in the train data: \n",
      " Atrophy / thinning of retinal layers     0.980\n",
      "Disruption of EZ                         7.116\n",
      "DRIL                                     0.389\n",
      "IR hemorrhages                           3.894\n",
      "IR HRF                                  65.132\n",
      "Partially attached vitreous face        29.753\n",
      "Fully attached vitreous face            56.780\n",
      "Preretinal tissue/hemorrhage             9.385\n",
      "Vitreous debris                         29.538\n",
      "VMT                                      0.094\n",
      "DRT/ME                                  31.378\n",
      "Fluid (IRF)                             41.756\n",
      "Fluid (SRF)                              2.417\n",
      "Disruption of RPE                        0.121\n",
      "PED (serous)                             0.134\n",
      "SHRM                                     0.765\n",
      "dtype: float64\n",
      "Present percentages for each biomarker in the test data: \n",
      " Atrophy / thinning of retinal layers     4.745\n",
      "Disruption of EZ                         3.776\n",
      "DRIL                                     0.153\n",
      "IR hemorrhages                           4.235\n",
      "IR HRF                                  76.020\n",
      "Partially attached vitreous face        39.184\n",
      "Fully attached vitreous face            50.663\n",
      "Preretinal tissue/hemorrhage             5.510\n",
      "Vitreous debris                         32.449\n",
      "VMT                                      0.153\n",
      "DRT/ME                                  33.980\n",
      "Fluid (IRF)                             49.898\n",
      "Fluid (SRF)                              2.704\n",
      "Disruption of RPE                        0.051\n",
      "PED (serous)                             0.000\n",
      "SHRM                                     0.969\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Calculate the percentages of zero values for each biomarker in the train data\n",
    "print(\"Present percentages for each biomarker in the train data: \\n\", ((train_data[labels] == 1).sum() / len(train_data) * 100).round(3))\n",
    "\n",
    "# Calculate the percentages of zero values for each biomarker in the test data\n",
    "print(\"Present percentages for each biomarker in the test data: \\n\", ((test_data[labels] == 1).sum() / len(test_data) * 100).round(3))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e6c99f5",
   "metadata": {},
   "source": [
    "Only run this to store the split in a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a0a7b084",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save as .csv files\n",
    "# train_data.to_csv('OLIVES_Dataset_Labels/TEMP_BiomarkerLabel_train_data.csv', index=False)\n",
    "# test_data.to_csv('OLIVES_Dataset_Labels/TEMP_BiomarkerLabel_test_data.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3f37cb0",
   "metadata": {},
   "source": [
    "c) Create dataset & transformation <br>\n",
    "Different transformations for testing and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "98eb93d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim.lr_scheduler import ExponentialLR\n",
    "from torch.utils.data import Dataset, Subset, DataLoader\n",
    "from torchvision import transforms, models\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import accuracy_score, f1_score, classification_report\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# set the size of the image according to your model needs\n",
    "imageSize = 224 # ResNet works with 224x224 pixels\n",
    "\n",
    "# Custom Dataset\n",
    "class BiomarkerDataset(Dataset):\n",
    "    def __init__(self, label_file, transform=None, num_frames=0):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            label_file (str): Path to the CSV file.\n",
    "            transform (callable, optional): Transform to be applied on a sample.\n",
    "            num_frames (int): Number of adjacent frames to use in the input sequence (1 adjacent frame -> 3 consecutive images).\n",
    "        \"\"\"\n",
    "        self.data = pd.read_csv(label_file)\n",
    "        self.transform = transform\n",
    "        self.num_frames = num_frames\n",
    "        \n",
    "        # Normalization\n",
    "        self.bcva_mean = self.data.iloc[:, 19].mean()\n",
    "        self.bcva_std = self.data.iloc[:, 19].std()\n",
    "        self.cst_mean = self.data.iloc[:, 20].mean()\n",
    "        self.cst_std = self.data.iloc[:, 20].std()\n",
    "\n",
    "        \n",
    "        # Exclude indices which don't have enough adjacent images\n",
    "        self.valid_indices = self.data[(self.data.iloc[:, 1] > num_frames) & (self.data.iloc[:, 1] < (50-num_frames))].index.tolist()\n",
    "\n",
    "    def __len__(self):\n",
    "        # we can't use the length of the data since we have to exclude the first and last image (for num_frames=1) of each OCT scan\n",
    "        return len(self.valid_indices)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        # Base path\n",
    "        img_base_path = '/storage/ice1/shared/d-pace_community/makerspace-datasets/MEDICAL/OLIVES/OLIVES'\n",
    "        \n",
    "        # Get the actual data index\n",
    "        index = self.valid_indices[idx]\n",
    "        \n",
    "        # Initialize\n",
    "        images = []\n",
    "        \n",
    "        # Load a sequence of consecutive images\n",
    "        for i in range(index - self.num_frames, index + self.num_frames +1):\n",
    "            img_path = img_base_path + self.data.iloc[i, 0]\n",
    "            img = Image.open(img_path).convert(\"L\") # 'L' is for grayscale; can be removed!?\n",
    "            \n",
    "            if self.transform is not None:\n",
    "                # apply data transformations (transforms it to tensor)\n",
    "                img = self.transform(img)\n",
    "            \n",
    "            # stack torch tensor\n",
    "            img = img.squeeze(0)  # Removes the first dimension if it's 1\n",
    "            images.append(img)\n",
    "        \n",
    "        # Stack the 3 grayscale images along the channel dimension\n",
    "        # Resulting tensor shape will be [3, H, W]\n",
    "        images = torch.stack(images, dim=0)\n",
    "        # print(images.shape) # debugging\n",
    "        \n",
    "        # Biomarker columns\n",
    "        labels = torch.tensor(self.data.iloc[index, 2:18].astype(float), dtype=torch.float32)\n",
    "        \n",
    "        # Get clinical data:\n",
    "        eye_id = self.data.iloc[index, 18]\n",
    "        bcva = self.data.iloc[index, 19]\n",
    "        cst = self.data.iloc[index, 20]\n",
    "        patient_id = self.data.iloc[index, 21]\n",
    "        \n",
    "        # Normalize\n",
    "        bcva = (bcva - self.bcva_mean) / self.bcva_std\n",
    "        cst = (cst - self.cst_mean) / self.cst_std\n",
    "\n",
    "        # Convert clinical data to tensor\n",
    "        clinical_data = torch.tensor([eye_id, bcva, cst, patient_id], dtype=torch.float32)\n",
    "        \n",
    "        return images, labels, clinical_data\n",
    "    \n",
    "    \n",
    "# Define transformers\n",
    "\n",
    "# Values for normalization taken from example paper\n",
    "mean = 0.1706\n",
    "std = 0.2112\n",
    "\n",
    "# train with data augmentation\n",
    "train_transformer = transforms.Compose([\n",
    "    # WORSE PERFORMANCE # transforms.RandomPerspective(distortion_scale=0.1, p=0.5, fill=0),  # Add perspective shift\n",
    "    # WORSE PERFORMANCE # transforms.RandomResizedCrop(size=imageSize, scale=(0.9, 1.0)), # RandomCrop between 70% to 100% of original size\n",
    "    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),  # Adjust color properties\n",
    "    transforms.RandomHorizontalFlip(p=0.5),  # Random horizontal flip\n",
    "    transforms.RandomRotation(degrees=10, fill=0),  # Rotates randomly between + and - degree and fills new pixels with black\n",
    "    transforms.Resize(imageSize), # Resize to models needs\n",
    "    transforms.ToTensor(),  # Convert image to tensor\n",
    "    transforms.Normalize(mean, std) # we have to calculate these values for our dataset\n",
    "])\n",
    "# train without data augmentation\n",
    "test_transformer = transforms.Compose([   \n",
    "    transforms.Resize(imageSize), # Resize to models needs\n",
    "    transforms.CenterCrop(imageSize), # shouldn't do anything\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean, std)\n",
    "])\n",
    "\n",
    "\n",
    "\n",
    "# set up train loader (just example since cross validation uses new ones)\n",
    "train_dataset = BiomarkerDataset(label_file='OLIVES_Dataset_Labels/BiomarkerLabel_train_data.csv', transform=train_transformer, num_frames=1)\n",
    "trainloader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=16, drop_last=True, pin_memory=True)\n",
    "\n",
    "# set up test loader (this one actually is being used)\n",
    "test_dataset = BiomarkerDataset(label_file='OLIVES_Dataset_Labels/BiomarkerLabel_train_data.csv', transform=test_transformer, num_frames=1)\n",
    "testloader = DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=16, pin_memory=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70b76d09",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}