DATA_CLEANING for RSF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Omar\\AppData\\Local\\Temp\\ipykernel_14224\\381285905.py:1: DtypeWarning: Columns (3,6,9,18,19,20,21,22,23,24,28,29,30,31,32,34,35,36,37,38,39,45,46,47,48,49,52,53,54,55,66,67,70,72,74,75,76,77,78,79,82,83,84,85,86,87,88,89,90,91,92,93,94,103,104,106,112,115,116,117,120,122,123,132,134,135,136,138,140,141,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,169,171,173,175,176,177,178,179,180,181,182,183,184,188,189,190,191,193,199,200,202,203,204,207,208,209,210,211,212,213,214,216,217,222,223,224,225,226,227,228,234,236,237,242,243,248,253,255,260,261,262,263,264,266,267,268,269,270,271,277,280,281,283,285,287,289,290,294,295,296,297,299,300,301,302,303,306,308,317,321,328,330,332,347,352,357,358,365,366,367,368,372,379,382,386,387,388,393,410,411,412,413,414,421,422,429,430,431,432,433,434,435,436,438,442,455,456,457,458,459,468,469,470,471) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  PAN_data = pd.read_csv(\"PAN_data.csv\")\n"
     ]
    }
   ],
   "source": [
    "PAN_data = pd.read_csv(\"PAN_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_remove = [\"WL_ORG\", \"COD_WL\", \"DONATION\", \"A2A2B_ELIGIBILITY\", \"ANTIBODY_TESTED\", \"ETHNICITY\", \"CITIZENSHIP\", \n",
    "             \"CITIZEN_COUNTRY\", \"PERM_STATE\", \"INIT_CURRENT_PRA\", \"INIT_PEAK_PRA\", \"INIT_STAT\", \"INIT_EPTS\", \n",
    "             \"END_EPTS\", \"ACTIVATE_DATE\", \"CREAT_CLEAR_DATE\", \"DEATH_DATE\", \"INIT_DATE\", \"WT_QUAL_DATE\", \"PT_CODE\", \n",
    "             \"DAYSWAIT_ALLOC\", \"WLHR\", \"WLHL\", \"WLIN\", \"WLKI\", \"WLKP\", \"WLLI\", \"WLLU\", \"WLPA\", \"WLPI\", \"WLVC\", \"REGION\", \n",
    "             \"INACT_REASON_CD\", \"WL_ID_CODE\", \"YR_ENTRY_US_TCR\", \"WORK_INCOME_TCR\", \"PRI_PAYMENT_TCR_KI\", \n",
    "             \"PRI_PAYMENT_TCR_PA\", \"PERM_STATE_TRR\", \"PRI_PAYMENT_TRR_KI\", \"PRI_PAYMENT_CTRY_TRR_KI\", \"FIRST_WK_DIAL\", \n",
    "             \"SERUM_CREAT\", \"RESUM_MAINT_DIAL_DT\", \"CMV_OLD_LIV_DON\", \"CMV_TEST_DON\", \"EBV_TEST_DON\", \"HBV_TEST_DON\", \n",
    "             \"HCV_TEST_DON\", \"CITIZENSHIP_DON\", \"HOME_STATE_DON\", \"CITIZEN_COUNTRY_DON\", \"RETXDATE_KI\", \"FAILDATE_KI\", \n",
    "             \"RESUM_MAINT_DIAL\", \"GRF_FAIL_CAUSE_TY_KI\", \"DWFG_KI\", \"PRVTXDIF_KI\", \"GTIME_KI\", \"GSTATUS_KI\", \"COD_KI\", \n",
    "             \"COD2_KI\", \"COD3_KI\", \"DAYSWAIT_CHRON_KI\", \"TX_PROCEDUR_TY_KI\", \"TRTREJ1Y_KI\", \"TRTREJ6M_KI\", \n",
    "             \"PRI_PAYMENT_CTRY_TRR_PA\", \"ACUTE_REJ_EPI_PA\", \"INSULIN_RESUMED_DATE_PA\", \"BLOOD_SUGAR_MED_RESUMED_DATE_PA\", \n",
    "             \"PRI_PAYMENT_CTRY_TCR_PA\", \"ENTERIC_DRAIN\", \"ENTERIC_DRAIN_DT\", \"RETXDATE_PA\", \n",
    "             \"PRVTXDIF_PA\", \"COD_PA\", \"COD2_PA\", \"COD3_PA\", \"TRTREJ1Y_PA\", \"TRTREJ6M_PA\", \n",
    "             \"PREV_KI_DATE\", \"FUNC_STAT_TRF\", \"SHARE_TY\", \"PSTATUS\", \"PTIME\", \"LOS\", \"PAYBACK\", \"ECD_DONOR\", \"STATUS_TCR\", \n",
    "             \"STATUS_TRR\", \"STATUS_DDR\", \"VAL_DT_DDR\", \"STATUS_LDR\", \"VAL_DT_LDR\", \"VAL_DT_TCR\", \"VAL_DT_TRR\", \n",
    "             \"RIGHTKIDNEYCHECKINDATE\", \"RIGHTKIDNEYCHECKINTIME\", \"RIGHTKIDNEYCHECKINTIMEZONEID\", \n",
    "             \"LEFTKIDNEYCHECKINDATE\", \"LEFTKIDNEYCHECKINTIME\", \"LEFTKIDNEYCHECKINTIMEZONEID\", \"ENBLOCKIDNEYCHECKINDATE\", \n",
    "             \"ENBLOCKIDNEYCHECKINTIME\", \"ENBLOCKIDNEYCHECKINTIMEZONEID\", \"PANCREASCHECKINDATE\", \"PANCREASCHECKINTIME\", \n",
    "             \"PANCREASCHECKINTIMEZONEID\", \"LT_ONE_WEEK_DON\", \"REJ_BIOPSY\", \"REJCNF_KI\", \"REJTRT_KI\", \"REJCNF_PA\", \n",
    "             \"REJTRT_PA\", \"DISCHARGE_DATE\", \"COMPL_ABSC\", \"COMPL_ANASLK\", \"COMPL_PANCREA\", \"SURG_INCIS\", \n",
    "             \"HASHBVVACCINATION\", \"REASONNOHBVVACCINATIONID\", \"EDUCATION_DON\", \"PRI_PAYMENT_DON\", \"PRI_PAYMENT_CTRY_DON\", \n",
    "             \"MEDICARE_DON\", \"MEDICAID_DON\", \"OTH_GOVT_DON\", \"PRIV_INS_DON\", \"HMO_PPO_DON\", \"SELF_DON\", \"DONATION_DON\", \n",
    "             \"FREE_DON\", \"RECOV_OUT_US\", \"RECOV_COUNTRY\", \"REFERRAL_DATE\", \"RECOVERY_DATE\", \"ADMIT_DATE_DON\", \n",
    "             \"TRANSPLANT_TIME\", \"TRANSPLANTTIMEZONEID\", \"DATA_TRANSPLANT\", \"DATA_WAITLIST\", \"CTR_CODE\", \"OPO_CTR_CODE\", \n",
    "             \"INIT_OPO_CTR_CODE\", \"END_OPO_CTR_CODE\", \"LISTING_CTR_CODE\", \"MAX_KDPI_LOCAL_ZERO_ABDR\", \n",
    "             \"MAX_KDPI_LOCAL_NON_ZERO_ABDR\", \"MAX_KDPI_IMPORT_ZERO_ABDR\", \"MAX_KDPI_IMPORT_NON_ZERO_ABDR\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_good = PAN_data.drop(columns=to_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the reason of removal to numeric and only include those that underwent transplantation [2,3,4,14,15,18,19]\n",
    "clean_good[\"REM_CD\"] = clean_good[\"REM_CD\"].replace(to_replace=\".\", value=0)\n",
    "clean_good[\"REM_CD\"] = pd.to_numeric(clean_good[\"REM_CD\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_good = clean_good[clean_good[\"REM_CD\"].isin([2,3,4,14,15,18,19])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TRR_ID_CODE</th>\n",
       "      <th>NUM_PREV_TX</th>\n",
       "      <th>CURRENT_PRA</th>\n",
       "      <th>PEAK_PRA</th>\n",
       "      <th>USE_WHICH_PRA</th>\n",
       "      <th>CREAT_CLEAR</th>\n",
       "      <th>GFR</th>\n",
       "      <th>ON_DIALYSIS</th>\n",
       "      <th>C_PEPTIDE</th>\n",
       "      <th>C_PEPTIDEDATE</th>\n",
       "      <th>...</th>\n",
       "      <th>RT_KI_GLOMERUL</th>\n",
       "      <th>DONOR_ID</th>\n",
       "      <th>HBSAB_DON</th>\n",
       "      <th>EBV_IGG_CAD_DON</th>\n",
       "      <th>EBV_IGM_CAD_DON</th>\n",
       "      <th>HBV_DNA_DON</th>\n",
       "      <th>CDC_RISK_HIV_DON</th>\n",
       "      <th>INO_PROCURE_AGENT_1</th>\n",
       "      <th>INO_PROCURE_AGENT_2</th>\n",
       "      <th>INO_PROCURE_AGENT_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A313794</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>C</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>N</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>264798</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A145568</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>251149</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A33519</td>\n",
       "      <td>2</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>265259</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A366169</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>18891</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A105490</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>104588</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37499</th>\n",
       "      <td>A1015280</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>Y</td>\n",
       "      <td>6.40</td>\n",
       "      <td>02/07/2024</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>691441</td>\n",
       "      <td>ND</td>\n",
       "      <td>P</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37500</th>\n",
       "      <td>A1014962</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.10</td>\n",
       "      <td>12/08/2023</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>691190</td>\n",
       "      <td>N</td>\n",
       "      <td>P</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Y</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37501</th>\n",
       "      <td>A1013505</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.12</td>\n",
       "      <td>02/01/2024</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>690455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Y</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37502</th>\n",
       "      <td>A1015290</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>19.0000</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.10</td>\n",
       "      <td>04/10/2023</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>691600</td>\n",
       "      <td>ND</td>\n",
       "      <td>P</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>4</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37503</th>\n",
       "      <td>A1015760</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.20</td>\n",
       "      <td>01/09/2024</td>\n",
       "      <td>...</td>\n",
       "      <td>.</td>\n",
       "      <td>692509</td>\n",
       "      <td>ND</td>\n",
       "      <td>P</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>37277 rows × 321 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      TRR_ID_CODE NUM_PREV_TX CURRENT_PRA PEAK_PRA USE_WHICH_PRA CREAT_CLEAR  \\\n",
       "0         A313794           1           0        0             C           .   \n",
       "1         A145568           0           .        .           NaN           .   \n",
       "2          A33519           2           .        .           NaN           .   \n",
       "3         A366169           0           .        .           NaN           .   \n",
       "4         A105490           0           .        .           NaN           .   \n",
       "...           ...         ...         ...      ...           ...         ...   \n",
       "37499    A1015280           0           .        .           NaN           .   \n",
       "37500    A1014962           0           .        .           NaN           .   \n",
       "37501    A1013505           0           .        .           NaN           .   \n",
       "37502    A1015290           0           .        .           NaN           .   \n",
       "37503    A1015760           0           .        .           NaN           .   \n",
       "\n",
       "           GFR ON_DIALYSIS C_PEPTIDE C_PEPTIDEDATE  ... RT_KI_GLOMERUL  \\\n",
       "0            .           N         .             .  ...              .   \n",
       "1            .         NaN         .             .  ...              .   \n",
       "2            .         NaN         .             .  ...              .   \n",
       "3            .         NaN         .             .  ...              .   \n",
       "4            .         NaN         .             .  ...              .   \n",
       "...        ...         ...       ...           ...  ...            ...   \n",
       "37499        .           Y      6.40    02/07/2024  ...              .   \n",
       "37500        .           Y      0.10    12/08/2023  ...              .   \n",
       "37501        .           Y      0.12    02/01/2024  ...              .   \n",
       "37502  19.0000           Y      0.10    04/10/2023  ...              .   \n",
       "37503   5.0000           Y      0.20    01/09/2024  ...              .   \n",
       "\n",
       "      DONOR_ID HBSAB_DON EBV_IGG_CAD_DON EBV_IGM_CAD_DON HBV_DNA_DON  \\\n",
       "0       264798       NaN             NaN             NaN         NaN   \n",
       "1       251149       NaN             NaN             NaN         NaN   \n",
       "2       265259       NaN             NaN             NaN         NaN   \n",
       "3        18891       NaN             NaN             NaN         NaN   \n",
       "4       104588       NaN             NaN             NaN         NaN   \n",
       "...        ...       ...             ...             ...         ...   \n",
       "37499   691441        ND               P               N         NaN   \n",
       "37500   691190         N               P               N         NaN   \n",
       "37501   690455       NaN               P               N         NaN   \n",
       "37502   691600        ND               P               N         NaN   \n",
       "37503   692509        ND               P               N         NaN   \n",
       "\n",
       "      CDC_RISK_HIV_DON INO_PROCURE_AGENT_1 INO_PROCURE_AGENT_2  \\\n",
       "0                  NaN                   .                   .   \n",
       "1                  NaN                   .                   .   \n",
       "2                  NaN                   .                   .   \n",
       "3                  NaN                   .                   .   \n",
       "4                  NaN                   .                   .   \n",
       "...                ...                 ...                 ...   \n",
       "37499                N                   .                   .   \n",
       "37500                Y                   .                   .   \n",
       "37501                Y                   .                   .   \n",
       "37502                N                   4                   .   \n",
       "37503                N                   .                   .   \n",
       "\n",
       "      INO_PROCURE_AGENT_3  \n",
       "0                       .  \n",
       "1                       .  \n",
       "2                       .  \n",
       "3                       .  \n",
       "4                       .  \n",
       "...                   ...  \n",
       "37499                   .  \n",
       "37500                   .  \n",
       "37501                   .  \n",
       "37502                   .  \n",
       "37503                   .  \n",
       "\n",
       "[37277 rows x 321 columns]"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "#extra removals\n",
    "to_remove_extra = [\"CURRENT_PRA\", \"PEAK_PRA\", \"USE_WHICH_PRA\", \"CREAT_CLEAR\", \"GFR\", \"C_PEPTIDE\", \"C_PEPTIDEDATE\", \n",
    "                   \"WGT_KG_TCR\", \"HGT_CM_TCR\", \"BMI_TCR\", \"TOT_SERUM_ALBUM\", \"C_PEPTIDE_PA_TCR\", \"HBA1C_PA_TCR\", \n",
    "                   \"INIT_WGT_KG\", \"INIT_HGT_CM\", \"INIT_CPRA\", \"END_CPRA\", \"END_STAT\", \"INIT_AGE\", \"DIALYSIS_DATE\", \n",
    "                   \"GFR_DATE\", \"INIT_BMI_CALC\", \"A1\", \"A2\", \"B1\", \"B2\", \"DR1\", \"DR2\", \"EDUCATION\", \"FUNC_STAT_TCR\", \n",
    "                   \"BW4\", \"BW6\", \"C1\", \"C2\", \"DR51\", \"DR51_2\", \"DR52\", \"DR52_2\", \"DR53\", \"DR53_2\", \"DQ1\", \"DQ2\", \n",
    "                   \"ACADEMIC_PRG_TCR\", \"ACADEMIC_LEVEL_TCR\", \"PREV_TX\", \"PREV_KI_TX\", \"PREV_PA_TX\", \"ACUTE_REJ_EPI_KI\", \n",
    "                   \"ORG_REC_ON\", \"L_FIN_FLOW_RATE_TX\", \"L_FIN_RESIST_TX\", \"R_FIN_FLOW_RATE_TX\", \"R_FIN_RESIST_TX\", \n",
    "                   \"FIN_RESIST_TX\", \"TXHRT\", \"TXINT\", \"TXKID\", \"TXLIV\", \"TXLNG\", \"TXPAN\", \"TXVCA\", \"RDA1\", \"RDA2\", \n",
    "                   \"RDB1\", \"RDB2\", \"RDDR1\", \"RDDR2\", \"DON_RETYP\", \"DA1\", \"DA2\", \"DB1\", \"DB2\", \"DDR1\", \"DDR2\", \"RA1\", \n",
    "                   \"RA2\", \"RB1\", \"RB2\", \"RDR1\", \"RDR2\", \"END_CPRA_DETAIL\", \"HAPLO_TY_MATCH_DON\", \"CMV_NUCLEIC_DON\", \n",
    "                   \"CMV_IGG_DON\", \"CMV_IGM_DON\", \"EBV_DNA_DON\", \"EBV_IGG_DON\", \"EBV_IGM_DON\", \n",
    "                   \"HCV_RNA_DON\", \"WARM_ISCH_TM_DON\", \"HCV_RIBA_DON\", \"HCV_ANTIBODY_DON\", \"CONTROLLED_DON\", \n",
    "                   \"CORE_COOL_DON\", \"HTLV1_OLD_DON\", \"HTLV2_OLD_DON\", \"DOBUT_DON_OLD\", \"DOPAMINE_DON_OLD\", \n",
    "                   \"PRETREAT_MED_DON_OLD\", \"CONTIN_ALCOHOL_OLD_DON\", \"CONTIN_OTH_DRUG_DON\", \"HIST_ALCOHOL_OLD_DON\", \n",
    "                   \"DIABDUR_DON\", \"HIST_IV_DRUG_OLD_DON\", \"INSULIN_DEP_DON\", \"OTHER_HYPERTENS_MED_DON\", \n",
    "                   \"HIST_INSULIN_DEP_DON\", \"INSULIN_DUR_DON\", \"HBV_NAT_DON\", \"HCV_NAT_DON\", \"HIV_NAT_DON\", \n",
    "                   \"CREAT6M\", \"CREAT1Y\", \"DIAL_DATE\", \"PRI_PAYMENT_TRR_PA\", \"PRE_AVG_INSULIN_USED_TRR\", \n",
    "                   \"PRE_AVG_INSULIN_USED_OLD_TRR\", \"INSULIN_PA\", \"INSULIN_DOSAGE_PA\", \"INSULIN_DURATION_PA\", \n",
    "                   \"METHOD_BLOOD_SUGAR_CONTROL_PA\", \"BLOOD_SUGAR_MEDICATION_PA\", \"BLOOD_SUGAR_DIET_PA\",\n",
    "                   \"C_PEPTIDE_PA_TRR\", \"HBA1C_PA_TRR\", \"INSULIN_DOSAGE_OLD_PA\", \"PK_DA1\", \"PK_DA2\", \"PK_DB1\", \n",
    "                   \"PK_DB2\", \"PK_DDR1\", \"PK_DDR2\", \"GRF_VASC_THROMB_PA\", \"INFECT_PA\", \"BLEED_PA\", \"ANAST_LK_PA\", \n",
    "                   \"REJ_ACUTE_PA\", \"REJ_HYPER_PA\", \"BIOP_ISLET_PA\", \"PANCREATIT_PA\", \"REJ_CHRONIC_PA\", \n",
    "                   \"PX_NON_COMPL_PA\", \"HBV_SURF_TOTAL\", \"HIV_NAT\", \"HCV_NAT\", \"HBV_NAT\", \"PREV_TX_ANY\", \"PREV_TX_ANY_N\", \n",
    "                   \"KI_CREAT_PREOP\", \"KI_PROC_TY\", \"TATTOOS\", \"LT_KI_GLOMERUL\", \"LT_KI_BIOPSY\", \"RT_KI_BIOPSY\", \n",
    "                   \"RT_KI_GLOMERUL\", \"HBV_DNA_DON\", \"CDC_RISK_HIV_DON\", \"INO_PROCURE_AGENT_1\", \"INO_PROCURE_AGENT_2\", \n",
    "                   \"INO_PROCURE_AGENT_3\", \"ACADEMIC_LEVEL_TRR\", \"ACADEMIC_PRG_TRR\", \"WORK_INCOME_TRR\", \n",
    "                   \"PRI_PAYMENT_CTRY_TCR_KI\", \"DONOR_ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37139, 321)\n"
     ]
    }
   ],
   "source": [
    "#remove very recent transplants\n",
    "\n",
    "clean_good = clean_good[clean_good[\"PX_STAT_DATE\"] != \".\"]\n",
    "print(clean_good.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Omar\\AppData\\Local\\Temp\\ipykernel_14224\\3346024158.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  clean_good[\"PX_STAT_DATE\"] = pd.to_datetime(clean_good[\"PX_STAT_DATE\"], dayfirst=False)\n",
      "C:\\Users\\Omar\\AppData\\Local\\Temp\\ipykernel_14224\\3346024158.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  clean_good[\"TX_DATE\"] = pd.to_datetime(clean_good[\"TX_DATE\"], dayfirst=False)\n"
     ]
    }
   ],
   "source": [
    "clean_good[\"PX_STAT_DATE\"] = pd.to_datetime(clean_good[\"PX_STAT_DATE\"], dayfirst=False)\n",
    "clean_good[\"TX_DATE\"] = pd.to_datetime(clean_good[\"TX_DATE\"], dayfirst=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Omar\\AppData\\Local\\Temp\\ipykernel_14224\\4063916721.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  clean_good[\"time_frame\"] = clean_good[\"PX_STAT_DATE\"] - clean_good[\"TX_DATE\"]\n"
     ]
    }
   ],
   "source": [
    "clean_good[\"time_frame\"] = clean_good[\"PX_STAT_DATE\"] - clean_good[\"TX_DATE\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Omar\\AppData\\Local\\Temp\\ipykernel_14224\\2960106135.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  clean_good[\"time_frame\"] = clean_good[\"time_frame\"].dt.days\n"
     ]
    }
   ],
   "source": [
    "#only include those with more than 5 years of followup\n",
    "clean_good[\"time_frame\"] = clean_good[\"time_frame\"].dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "failed_good = clean_good.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "cutoff = pd.to_datetime(\"04/04/2019\", dayfirst=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove transplants with less than 5 years of followup time\n",
    "#clean_good = clean_good[clean_good[\"TX_DATE\"] < cutoff]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(37139, 322)"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_good.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "very_clean_data = clean_good.drop(columns=to_remove_extra)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "very_clean_data = very_clean_data[very_clean_data[\"AGE_GROUP\"] == \"A\"]\n",
    "\n",
    "very_clean_data = very_clean_data[very_clean_data[\"DON_TY\"] == \"C\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_extra_to_remove = [\"NUM_PREV_TX\", \"DRUGTRT_COPD\", \"REM_CD\", \"END_BMI_CALC\", \"MALIG_TCR_KI\", \"MALIG_TCR_PA\", \n",
    "                         \"PREV_MALIG_TY\", \"LIV_DON_TY\", \"EXTRACRANIAL_CANCER_DON\", \n",
    "                         \"INTRACRANIAL_CANCER_DON\", \"SKIN_CANCER_DON\", \"HIST_DIABETES_DON\", \n",
    "                         \"ON_DIALYSIS\", \"END_STAT_PA\", \"ADMISSION_DATE\", \"END_STAT_KI\", \"END_DATE\", \"DIABETES_DON\", \n",
    "                         \"DGN2_TCR\", \"EXH_PERIT_ACCESS\", \"EXH_VASC_ACCESS\", \"REC_ON_ICE\", \"REC_ON_PUMP\", \"PRE_TX_TXFUS\", \n",
    "                         \"DON_TY\", \"CONTIN_CIG_DON\", \"CONTIN_IV_DRUG_OLD_DON\", \"CONTIN_COCAINE_DON\", \"DIET_DON\", \n",
    "                         \"DIURETICS_DON\", \"KDPI\", \"KDRI_MED\", \"KDRI_RAO\", \"PUMP_KI\", \"DIAG_KI\", \"COLD_ISCH_KI\", \n",
    "                         \"GRF_STAT_KI\", \"GSTATUS_PA\", \"INOTROP_AGENTS\", \"HBSAB_DON\", \"EBV_IGG_CAD_DON\", \n",
    "                         \"EBV_IGM_CAD_DON\", \"DAYSWAIT_CHRON\", \"FUNC_STAT_TRR\", \"MALIG_TRR\" ,\"MALIG_TY_TRR\", \n",
    "                         \"PREV_PREG\", \"HBV_CORE_DON\", \"HEP_C_ANTI_DON\", \"BLOOD_INF_DON\", \"OTHER_INF_DON\", \"PULM_INF_DON\", \n",
    "                         \"URINE_INF_DON\", \"VDRL_DON\", \"HYPERTENS_DUR_DON\", \"CANCER_FREE_INT_DON\", \"ARGININE_DON\", \n",
    "                         \"INSULIN_DON\", \"DISTANCE\", \"GRF_FAIL_CAUSE_TY_PA\", \"AGE_GROUP\", \"HIV_SEROSTATUS\", \"OPER_TECH\", \n",
    "                         \"HBV_SUR_ANTIGEN_DON\", \"CANCER_SITE_DON\", \"MALIG_TY\", \"DGN_TCR\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pre_imp = very_clean_data.drop(columns=final_extra_to_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "# many variables were first collected in 1994 and the transplants done prior to this account for most of the missing data\n",
    "# in this dataset\n",
    "after_94 = pd.to_datetime(\"04/01/2004\", dayfirst=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pre_imp = data_pre_imp[data_pre_imp[\"TX_DATE\"] > after_94]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pre_imp = data_pre_imp[data_pre_imp[\"HLAMIS\"] != \".\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21153, 90)"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pre_imp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_pre_imp[\"5YR_SURV\"] = pd.Series(dtype=\"int\")\n",
    "# data_pre_imp[\"index\"] = range(33004)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16239, 90)\n",
      "(16236, 90)\n"
     ]
    }
   ],
   "source": [
    "#grafts that survived five years\n",
    "surv = data_pre_imp[data_pre_imp[\"GRF_STAT_PA\"] == \"Y\"]\n",
    "print(surv.shape)\n",
    "surv = surv[surv[\"FAILDATE_PA\"] == \".\"]\n",
    "print(surv.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4914, 90)\n",
      "(4909, 90)\n",
      "(4909, 90)\n",
      "(4909, 91)\n",
      "(4909, 91)\n",
      "(4909, 90)\n"
     ]
    }
   ],
   "source": [
    "#grafts that failed in the first five years but here we are using the dataset including all transplants \n",
    "# because if a transplant done in 2021 failed in 1 year it should still be taken into account\n",
    "fail = data_pre_imp[data_pre_imp[\"GRF_STAT_PA\"] == \"N\"]\n",
    "print(fail.shape)\n",
    "#drop data wihtout fail date\n",
    "fail = fail[fail[\"FAILDATE_PA\"] != \".\"]\n",
    "print(fail.shape)\n",
    "fail[\"FAILDATE_PA\"] = pd.to_datetime(fail[\"FAILDATE_PA\"], dayfirst=False)\n",
    "print(fail.shape)\n",
    "fail[\"fail_frame\"] = fail[\"FAILDATE_PA\"] - fail[\"TX_DATE\"] \n",
    "print(fail.shape)\n",
    "fail[\"fail_frame\"] = fail[\"fail_frame\"].dt.days\n",
    "print(fail.shape)\n",
    "fail[\"time_frame\"] = fail[\"fail_frame\"]\n",
    "fail = fail.drop(columns=[\"fail_frame\"], axis=1)\n",
    "print(fail.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "# failed_after_5 = failed_after_5.drop(\"fail_frame\", axis=1)\n",
    "# fail = fail.drop(\"fail_frame\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [surv, fail]\n",
    "labelled_pre_imp = pd.concat(frames)\n",
    "labelled_pre_imp = data_pre_imp.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labelled_pre_imp = labelled_pre_imp.sort_values(['index'])\n",
    "no_longer_needed = [\"PX_STAT_DATE\", \"PX_STAT\", \"GTIME_PA\", \"FAILDATE_PA\", \n",
    "                    \"TX_DATE\", \"COMPOSITE_DEATH_DATE\"]\n",
    "labelled_pre_imp = labelled_pre_imp.drop(columns=no_longer_needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove 86 donors with unkown height or weight or both\n",
    "labelled_pre_imp = labelled_pre_imp[labelled_pre_imp[\"BMI_DON_CALC\"] != \".\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation = labelled_pre_imp.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(imputation):\n",
    "    # Replace missing values with \"U\" in column: 'PERIP_VASC'\n",
    "    imputation = imputation.fillna({'PERIP_VASC': \"U\"})\n",
    "  # Change column type to int32 for column: 'AMIS'\n",
    "    imputation = imputation.astype({'AMIS': 'int32'})\n",
    "    # Change column type to int32 for columns: 'BMIS', 'DRMIS', 'HLAMIS'\n",
    "    imputation = imputation.astype({'BMIS': 'int32', 'DRMIS': 'int32', 'HLAMIS': 'int32'})\n",
    "    # Change column type to int32 for column: 'AGE_DON'\n",
    "    imputation = imputation.astype({'AGE_DON': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'DDAVP_DON'\n",
    "    imputation = imputation.fillna({'DDAVP_DON': \"U\"})\n",
    "    # Replace missing values with \"N\" in column: 'CMV_DON'\n",
    "    imputation = imputation.fillna({'CMV_DON': \"N\"})\n",
    "    # Replace all instances of \"ND\" with \"N\" in column: 'CMV_DON'\n",
    "    imputation['CMV_DON'] = imputation['CMV_DON'].str.replace(\"ND\", \"N\", case=False, regex=False)\n",
    "    # Replace all instances of \"I\" with \"N\" in column: 'CMV_DON'\n",
    "    imputation['CMV_DON'] = imputation['CMV_DON'].str.replace(\"I\", \"N\", case=False, regex=False)\n",
    "    # Replace all instances of \"U\" with \"N\" in column: 'CMV_DON'\n",
    "    imputation['CMV_DON'] = imputation['CMV_DON'].str.replace(\"U\", \"N\", case=False, regex=False)\n",
    "    # Replace all instances of \"C\" with \"N\" in column: 'CMV_DON'\n",
    "    imputation['CMV_DON'] = imputation['CMV_DON'].str.replace(\"C\", \"N\", case=False, regex=False)\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'CMV_DON'\n",
    "    imputation['CMV_DON'] = imputation['CMV_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"P\" with \"1\" in column: 'CMV_DON'\n",
    "    imputation['CMV_DON'] = imputation['CMV_DON'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'CMV_DON'\n",
    "    imputation = imputation.astype({'CMV_DON': 'int32'})\n",
    "    # Change column type to int32 for column: 'CMV_DON'\n",
    "    imputation = imputation.astype({'CMV_DON': 'int32'})\n",
    "    \n",
    "   # Replace missing values with \"N\" in column: 'NON_HRT_DON'\n",
    "    imputation = imputation.fillna({'NON_HRT_DON': \"N\"})\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'NON_HRT_DON'\n",
    "    imputation['NON_HRT_DON'] = imputation['NON_HRT_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'NON_HRT_DON'\n",
    "    imputation['NON_HRT_DON'] = imputation['NON_HRT_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'NON_HRT_DON'\n",
    "    imputation = imputation.astype({'NON_HRT_DON': 'int32'})\n",
    "    # Change column type to int32 for column: 'NON_HRT_DON'\n",
    "    imputation = imputation.astype({'NON_HRT_DON': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'ANTIHYPE_DON'\n",
    "    imputation = imputation.fillna({'ANTIHYPE_DON': \"U\"})\n",
    "   # Replace all instances of \"N\" with \"0\" in column: 'BLOOD_INF_CONF_DON'\n",
    "    imputation['BLOOD_INF_CONF_DON'] = imputation['BLOOD_INF_CONF_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace missing values with \"0\" in column: 'BLOOD_INF_CONF_DON'\n",
    "    imputation = imputation.fillna({'BLOOD_INF_CONF_DON': \"0\"})\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'BLOOD_INF_CONF_DON'\n",
    "    imputation['BLOOD_INF_CONF_DON'] = imputation['BLOOD_INF_CONF_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'BLOOD_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'BLOOD_INF_CONF_DON': 'int32'})\n",
    "    # Change column type to int32 for column: 'BLOOD_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'BLOOD_INF_CONF_DON': 'int32'})\n",
    "    # Change column type to string for column: 'BUN_DON'\n",
    "    imputation = imputation.astype({'BUN_DON': 'string'})\n",
    "    # Replace all instances of \".\" with \"\" in column: 'BUN_DON'\n",
    "    imputation.loc[imputation['BUN_DON'].str.lower() == \".\".lower(), 'BUN_DON'] = np.nan\n",
    "    # Change column type to float32 for column: 'BUN_DON'\n",
    "    imputation = imputation.astype({'BUN_DON': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'BUN_DON'\n",
    "    imputation = imputation.fillna({'BUN_DON': imputation['BUN_DON'].median()})\n",
    "    # Change column type to string for column: 'CREAT_DON'\n",
    "    imputation = imputation.astype({'CREAT_DON': 'string'})\n",
    "    # Replace all instances of \".\" with \"\" in column: 'CREAT_DON'\n",
    "    imputation.loc[imputation['CREAT_DON'].str.lower() == \".\".lower(), 'CREAT_DON'] = np.nan\n",
    "    # Change column type to float32 for column: 'CREAT_DON'\n",
    "    imputation = imputation.astype({'CREAT_DON': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'CREAT_DON'\n",
    "    imputation = imputation.fillna({'CREAT_DON': imputation['CREAT_DON'].median()})\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'OTHER_INF_CONF_DON'\n",
    "    imputation['OTHER_INF_CONF_DON'] = imputation['OTHER_INF_CONF_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace missing values with \"0\" in column: 'OTHER_INF_CONF_DON'\n",
    "    imputation = imputation.fillna({'OTHER_INF_CONF_DON': \"0\"})\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'OTHER_INF_CONF_DON'\n",
    "    imputation['OTHER_INF_CONF_DON'] = imputation['OTHER_INF_CONF_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'OTHER_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'OTHER_INF_CONF_DON': 'int32'})\n",
    "    # Change column type to int32 for column: 'OTHER_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'OTHER_INF_CONF_DON': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'PT_DIURETICS_DON'\n",
    "    imputation = imputation.fillna({'PT_DIURETICS_DON': \"U\"})\n",
    "   \n",
    "    # Replace missing values with \"U\" in column: 'PT_STEROIDS_DON'\n",
    "    imputation = imputation.fillna({'PT_STEROIDS_DON': \"U\"})\n",
    "   # Replace missing values with \"U\" in column: 'PT_T3_DON'\n",
    "    imputation = imputation.fillna({'PT_T3_DON': \"U\"})\n",
    "     # Replace missing values with \"U\" in column: 'PT_T4_DON'\n",
    "    imputation = imputation.fillna({'PT_T4_DON': \"U\"})\n",
    "   # Replace missing values with \"0\" in column: 'PULM_INF_CONF_DON'\n",
    "    imputation = imputation.fillna({'PULM_INF_CONF_DON': \"0\"})\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'PULM_INF_CONF_DON'\n",
    "    imputation['PULM_INF_CONF_DON'] = imputation['PULM_INF_CONF_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'PULM_INF_CONF_DON'\n",
    "    imputation['PULM_INF_CONF_DON'] = imputation['PULM_INF_CONF_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'PULM_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'PULM_INF_CONF_DON': 'int32'})\n",
    "    # Change column type to int32 for column: 'PULM_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'PULM_INF_CONF_DON': 'int32'})\n",
    "    # Change column type to string for columns: 'SGOT_DON', 'SGPT_DON', 'TBILI_DON'\n",
    "    imputation = imputation.astype({'SGOT_DON': 'string', 'SGPT_DON': 'string', 'TBILI_DON': 'string'})\n",
    "    # Replace all instances of \".\" with \"\" in columns: 'SGOT_DON', 'SGPT_DON', 'TBILI_DON'\n",
    "    imputation.loc[imputation['SGOT_DON'].str.lower() == \".\".lower(), 'SGOT_DON'] = np.nan\n",
    "    imputation.loc[imputation['SGPT_DON'].str.lower() == \".\".lower(), 'SGPT_DON'] = np.nan\n",
    "    imputation.loc[imputation['TBILI_DON'].str.lower() == \".\".lower(), 'TBILI_DON'] = np.nan\n",
    "    # Change column type to float32 for columns: 'SGOT_DON', 'SGPT_DON', 'TBILI_DON'\n",
    "    imputation = imputation.astype({'SGOT_DON': 'float32', 'SGPT_DON': 'float32', 'TBILI_DON': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'SGOT_DON', 'SGPT_DON', 'TBILI_DON'\n",
    "    imputation = imputation.fillna({'SGOT_DON': imputation['SGOT_DON'].median(), 'SGPT_DON': imputation['SGPT_DON'].median(), 'TBILI_DON': imputation['TBILI_DON'].median()})\n",
    "    # Replace missing values with \"0\" in column: 'URINE_INF_CONF_DON'\n",
    "    imputation = imputation.fillna({'URINE_INF_CONF_DON': \"0\"})\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'URINE_INF_CONF_DON'\n",
    "    imputation['URINE_INF_CONF_DON'] = imputation['URINE_INF_CONF_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'URINE_INF_CONF_DON'\n",
    "    imputation['URINE_INF_CONF_DON'] = imputation['URINE_INF_CONF_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'URINE_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'URINE_INF_CONF_DON': 'int32'})\n",
    "    # Change column type to int32 for column: 'URINE_INF_CONF_DON'\n",
    "    imputation = imputation.astype({'URINE_INF_CONF_DON': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'VASODIL_DON'\n",
    "    imputation = imputation.fillna({'VASODIL_DON': \"U\"})\n",
    "    # Replace missing values with \"U\" in column: 'CLIN_INFECT_DON'\n",
    "    imputation = imputation.fillna({'CLIN_INFECT_DON': \"U\"})\n",
    "    \n",
    "    # Replace missing values with \"U\" in column: 'HIST_CIG_DON'\n",
    "    imputation = imputation.fillna({'HIST_CIG_DON': \"U\"})\n",
    "    # Replace missing values with \"U\" in columns: 'HIST_COCAINE_DON', 'HIST_HYPERTENS_DON', 'HIST_OTH_DRUG_DON'\n",
    "    imputation = imputation.fillna({'HIST_COCAINE_DON': \"U\", 'HIST_HYPERTENS_DON': \"U\", 'HIST_OTH_DRUG_DON': \"U\"})\n",
    " # Replace missing values with \"U\" in column: 'HEPARIN_DON'\n",
    "    imputation = imputation.fillna({'HEPARIN_DON': \"U\"})\n",
    "    # Change column type to int32 for column: 'ABO_MAT'\n",
    "    imputation = imputation.astype({'ABO_MAT': 'int32'})\n",
    "    # Change column type to int32 for column: 'AGE'\n",
    "    imputation = imputation.astype({'AGE': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'DIAL_TRR'\n",
    "    imputation = imputation.fillna({'DIAL_TRR': \"U\"})\n",
    "     # Replace missing values with \"0\" in column: 'MULTIORG'\n",
    "    imputation = imputation.fillna({'MULTIORG': \"0\"})\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'MULTIORG'\n",
    "    imputation['MULTIORG'] = imputation['MULTIORG'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'MULTIORG'\n",
    "    imputation = imputation.astype({'MULTIORG': 'int32'})\n",
    "    # Change column type to string for column: 'ART_RECON'\n",
    "    imputation = imputation.astype({'ART_RECON': 'string'})\n",
    "    # Replace all instances of \".\" with \"2\" in column: 'ART_RECON'\n",
    "    imputation['ART_RECON'] = imputation['ART_RECON'].str.replace(\".\", \"999\", case=False, regex=False)\n",
    "    # Change column type to string for column: 'DUCT_MGMT'\n",
    "    imputation = imputation.astype({'DUCT_MGMT': 'string'})\n",
    "    # Replace all instances of \".\" with \"999\" in column: 'DUCT_MGMT'\n",
    "    imputation['DUCT_MGMT'] = imputation['DUCT_MGMT'].str.replace(\".\", \"999\", case=False, regex=False)\n",
    "    # Change column type to string for column: 'GRF_PLACEM'\n",
    "    imputation = imputation.astype({'GRF_PLACEM': 'string'})\n",
    "    # Replace all instances of \".\" with \"1\" in column: 'GRF_PLACEM'\n",
    "    imputation['GRF_PLACEM'] = imputation['GRF_PLACEM'].str.replace(\".\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \".\" with \"\" in column: 'PA_PRESERV_TM'\n",
    "    imputation.loc[imputation['PA_PRESERV_TM'].str.lower() == \".\".lower(), 'PA_PRESERV_TM'] = np.nan\n",
    "    # Change column type to float32 for column: 'PA_PRESERV_TM'\n",
    "    imputation = imputation.astype({'PA_PRESERV_TM': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'PA_PRESERV_TM'\n",
    "    imputation = imputation.fillna({'PA_PRESERV_TM': imputation['PA_PRESERV_TM'].median()})\n",
    "\n",
    "    # Change column type to string for column: 'VASC_MGMT'\n",
    "    imputation = imputation.astype({'VASC_MGMT': 'string'})\n",
    "    # Replace all instances of \".\" with \"1\" in column: 'VASC_MGMT'\n",
    "    imputation['VASC_MGMT'] = imputation['VASC_MGMT'].str.replace(\".\", \"1\", case=False, regex=False)\n",
    "    # Replace missing values with \"N\" in column: 'VEN_EXT_GRF'\n",
    "    imputation = imputation.fillna({'VEN_EXT_GRF': \"N\"})\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'VEN_EXT_GRF'\n",
    "    imputation['VEN_EXT_GRF'] = imputation['VEN_EXT_GRF'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'VEN_EXT_GRF'\n",
    "    imputation['VEN_EXT_GRF'] = imputation['VEN_EXT_GRF'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'VEN_EXT_GRF'\n",
    "    imputation = imputation.astype({'VEN_EXT_GRF': 'int32'})\n",
    "    # Change column type to string for column: 'DIAG_PA'\n",
    "    imputation = imputation.astype({'DIAG_PA': 'string'})\n",
    "    # Change column type to string for column: 'DAYSWAIT_CHRON_PA'\n",
    "    imputation = imputation.astype({'DAYSWAIT_CHRON_PA': 'string'})\n",
    "    # Replace all instances of \".\" with \"\" in column: 'DAYSWAIT_CHRON_PA'\n",
    "    imputation.loc[imputation['DAYSWAIT_CHRON_PA'].str.lower() == \".\".lower(), 'DAYSWAIT_CHRON_PA'] = np.nan\n",
    "    # Change column type to float32 for column: 'DAYSWAIT_CHRON_PA'\n",
    "    imputation = imputation.astype({'DAYSWAIT_CHRON_PA': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'DAYSWAIT_CHRON_PA'\n",
    "    imputation = imputation.fillna({'DAYSWAIT_CHRON_PA': imputation['DAYSWAIT_CHRON_PA'].median()})\n",
    "    # Change column type to int32 for column: 'DAYSWAIT_CHRON_PA'\n",
    "    imputation = imputation.astype({'DAYSWAIT_CHRON_PA': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'CMV_IGG'\n",
    "    imputation = imputation.fillna({'CMV_IGG': \"U\"})\n",
    "    # Replace all instances of \"ND\" with \"U\" in column: 'CMV_IGG'\n",
    "    imputation['CMV_IGG'] = imputation['CMV_IGG'].str.replace(\"ND\", \"U\", case=False, regex=False)\n",
    "    # Replace missing values with \"U\" in columns: 'CMV_IGM', 'HBV_CORE' and 2 other columns\n",
    "    imputation = imputation.fillna({'CMV_IGM': \"U\", 'HBV_CORE': \"U\", 'HCV_SEROSTATUS': \"U\", 'CMV_STATUS': \"U\"})\n",
    "    # Replace all instances of \"ND\" with \"U\" in columns: 'CMV_IGM', 'HBV_CORE' and 2 other columns\n",
    "    imputation['CMV_IGM'] = imputation['CMV_IGM'].str.replace(\"ND\", \"U\", case=False, regex=False)\n",
    "    imputation['HBV_CORE'] = imputation['HBV_CORE'].str.replace(\"ND\", \"U\", case=False, regex=False)\n",
    "    imputation['HCV_SEROSTATUS'] = imputation['HCV_SEROSTATUS'].str.replace(\"ND\", \"U\", case=False, regex=False)\n",
    "    imputation['CMV_STATUS'] = imputation['CMV_STATUS'].str.replace(\"ND\", \"U\", case=False, regex=False)\n",
    "    # Replace missing values with \"U\" in column: 'HBV_SUR_ANTIGEN'\n",
    "    imputation = imputation.fillna({'HBV_SUR_ANTIGEN': \"U\"})\n",
    "    # Replace all instances of \"ND\" with \"U\" in column: 'HBV_SUR_ANTIGEN'\n",
    "    imputation['HBV_SUR_ANTIGEN'] = imputation['HBV_SUR_ANTIGEN'].str.replace(\"ND\", \"U\", case=False, regex=False)\n",
    "    # Replace missing values with \"U\" in column: 'EBV_SEROSTATUS'\n",
    "    imputation = imputation.fillna({'EBV_SEROSTATUS': \"U\"})\n",
    "   # Change column type to string for column: 'MED_COND_TRR'\n",
    "    imputation = imputation.astype({'MED_COND_TRR': 'string'})\n",
    "    # Replace all instances of \".\" with \"U\" in column: 'MED_COND_TRR'\n",
    "    imputation['MED_COND_TRR'] = imputation['MED_COND_TRR'].str.replace(\".\", \"U\", case=False, regex=False)\n",
    "   # Change column type to string for columns: 'HGT_CM_CALC', 'WGT_KG_CALC', 'BMI_CALC'\n",
    "    imputation = imputation.astype({'HGT_CM_CALC': 'string', 'WGT_KG_CALC': 'string', 'BMI_CALC': 'string'})\n",
    "    # Replace all instances of \".\" with \"\" in column: 'HGT_CM_CALC'\n",
    "    imputation.loc[imputation['HGT_CM_CALC'].str.lower() == \".\".lower(), 'HGT_CM_CALC'] = np.nan\n",
    "    # Replace all instances of \".\" with \"\" in columns: 'WGT_KG_CALC', 'BMI_CALC'\n",
    "    imputation.loc[imputation['WGT_KG_CALC'].str.lower() == \".\".lower(), 'WGT_KG_CALC'] = np.nan\n",
    "    imputation.loc[imputation['BMI_CALC'].str.lower() == \".\".lower(), 'BMI_CALC'] = np.nan\n",
    "    # Change column type to float32 for columns: 'HGT_CM_CALC', 'WGT_KG_CALC', 'BMI_CALC'\n",
    "    imputation = imputation.astype({'HGT_CM_CALC': 'float32', 'WGT_KG_CALC': 'float32', 'BMI_CALC': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'HGT_CM_CALC', 'WGT_KG_CALC', 'BMI_CALC'\n",
    "    imputation = imputation.fillna({'HGT_CM_CALC': imputation['HGT_CM_CALC'].median(), 'WGT_KG_CALC': imputation['WGT_KG_CALC'].median(), 'BMI_CALC': imputation['BMI_CALC'].median()})\n",
    "    # Replace missing values with \"U\" in column: 'PROTEIN_URINE'\n",
    "    imputation = imputation.fillna({'PROTEIN_URINE': \"U\"})\n",
    "    # Replace missing values with \"U\" in column: 'CARDARREST_NEURO'\n",
    "    imputation = imputation.fillna({'CARDARREST_NEURO': \"U\"})\n",
    "   # Replace all instances of \".\" with \"0\" in column: 'RESUSCIT_DUR'\n",
    "    imputation.loc[imputation['RESUSCIT_DUR'].str.lower() == \".\".lower(), 'RESUSCIT_DUR'] = \"0\"\n",
    "    # Change column type to float32 for column: 'RESUSCIT_DUR'\n",
    "    imputation = imputation.astype({'RESUSCIT_DUR': 'float32'})\n",
    "    # Change column type to int32 for column: 'RESUSCIT_DUR'\n",
    "    imputation = imputation.astype({'RESUSCIT_DUR': 'int32'})\n",
    "    # Replace missing values with \"U\" in column: 'INOTROP_SUPPORT_DON'\n",
    "    imputation = imputation.fillna({'INOTROP_SUPPORT_DON': \"U\"})\n",
    "   \n",
    "    # Replace missing values with \"N\" in column: 'HIST_CANCER_DON'\n",
    "    imputation = imputation.fillna({'HIST_CANCER_DON': \"N\"})\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'HIST_CANCER_DON'\n",
    "    imputation['HIST_CANCER_DON'] = imputation['HIST_CANCER_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"U\" with \"0\" in column: 'HIST_CANCER_DON'\n",
    "    imputation['HIST_CANCER_DON'] = imputation['HIST_CANCER_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"1\" in column: 'HIST_CANCER_DON'\n",
    "    imputation['HIST_CANCER_DON'] = imputation['HIST_CANCER_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Change column type to int32 for column: 'HIST_CANCER_DON'\n",
    "    imputation = imputation.astype({'HIST_CANCER_DON': 'int32'})\n",
    "\n",
    "    # Change column type to int32 for column: 'ETHCAT'\n",
    "    imputation = imputation.astype({'ETHCAT': 'string'})\n",
    "    # Replace all instances of \".\" with \"\" in column: 'AGE_DIAB'\n",
    "    imputation.loc[imputation['AGE_DIAB'].str.lower() == \".\".lower(), 'AGE_DIAB'] = np.nan\n",
    "    # Change column type to float32 for column: 'AGE_DIAB'\n",
    "    imputation = imputation.astype({'AGE_DIAB': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'AGE_DIAB'\n",
    "    imputation = imputation.fillna({'AGE_DIAB': imputation['AGE_DIAB'].median()})\n",
    "    # Replace all instances of \".\" with \"\" in column: 'CREAT_TRR'\n",
    "    imputation.loc[imputation['CREAT_TRR'].str.lower() == \".\".lower(), 'CREAT_TRR'] = np.nan\n",
    "    # Change column type to float32 for column: 'CREAT_TRR'\n",
    "    imputation = imputation.astype({'CREAT_TRR': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'CREAT_TRR'\n",
    "    imputation = imputation.fillna({'CREAT_TRR': imputation['CREAT_TRR'].median()})\n",
    "    # Change column type to string for column: 'ETHCAT_DON'\n",
    "    imputation = imputation.astype({'ETHCAT_DON': 'string'})\n",
    "    # Change column type to float32 for columns: 'HGT_CM_DON_CALC', 'WGT_KG_DON_CALC', 'BMI_DON_CALC'\n",
    "    imputation = imputation.astype({'HGT_CM_DON_CALC': 'float32', 'WGT_KG_DON_CALC': 'float32', 'BMI_DON_CALC': 'float32'})\n",
    "    # Round column 'BMI_DON_CALC' (Number of decimals: 1)\n",
    "    imputation = imputation.round({'BMI_DON_CALC': 1})\n",
    "    # Change column type to string for column: 'DIAG_PA'\n",
    "    imputation = imputation.astype({'DIAG_PA': 'string'})\n",
    "    # Replace all instances of \".\" with \"5001\" in column: 'DIAG_PA'\n",
    "    imputation['DIAG_PA'] = imputation['DIAG_PA'].str.replace(\".\", \"5001\", case=False, regex=False)\n",
    "\n",
    "    # Replace all instances of \".\" with \"\" in columns: 'LIPASE', 'AMYLASE'\n",
    "    imputation.loc[imputation['LIPASE'].str.lower() == \".\".lower(), 'LIPASE'] = np.nan\n",
    "    imputation.loc[imputation['AMYLASE'].str.lower() == \".\".lower(), 'AMYLASE'] = np.nan\n",
    "    # Change column type to float32 for columns: 'LIPASE', 'AMYLASE'\n",
    "    imputation = imputation.astype({'LIPASE': 'float32', 'AMYLASE': 'float32'})\n",
    "    # Replace missing values with the median of each column in: 'LIPASE', 'AMYLASE'\n",
    "    imputation = imputation.fillna({'LIPASE': imputation['LIPASE'].median(), 'AMYLASE': imputation['AMYLASE'].median()})\n",
    "    \n",
    "\n",
    "    # Change column type to string for column: 'DIAB'\n",
    "    imputation = imputation.astype({'DIAB': 'string'})\n",
    "    # Replace all instances of \"1\" with \"0\" in column: 'DIAB'\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \"1\".lower(), 'DIAB'] = \"0\"\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \"998\".lower(), 'DIAB'] = \"0\"\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \".\".lower(), 'DIAB'] = \"0\"\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \"2\".lower(), 'DIAB'] = \"1\"\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \"3\".lower(), 'DIAB'] = \"1\"\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \"4\".lower(), 'DIAB'] = \"1\"\n",
    "    imputation.loc[imputation['DIAB'].str.lower() == \"5\".lower(), 'DIAB'] = \"1\"\n",
    "\n",
    "    imputation[\"DIAB\"] = imputation[\"DIAB\"].astype(\"float32\")\n",
    "\n",
    "    return imputation\n",
    "   \n",
    "\n",
    "\n",
    "imputation_clean_no_oh = clean_data(imputation.copy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def clean_data(imputation_clean_no_oh):\n",
    "    # Replace all instances of \"M\" with \"0\" in column: 'GENDER'\n",
    "    imputation_clean_no_oh['GENDER'] = imputation_clean_no_oh['GENDER'].str.replace(\"M\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"F\" with \"1\" in column: 'GENDER'\n",
    "    imputation_clean_no_oh['GENDER'] = imputation_clean_no_oh['GENDER'].str.replace(\"F\", \"1\", case=False, regex=False)\n",
    "    # Change column type to float32 for column: 'GENDER'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.astype({'GENDER': 'float32'})\n",
    "    # Replace all instances of \"A1\" with \"A\" in column: 'ABO'\n",
    "    imputation_clean_no_oh['ABO'] = imputation_clean_no_oh['ABO'].str.replace(\"A1\", \"A\", case=False, regex=False)\n",
    "    # Replace all instances of \"A2\" with \"\" in column: 'ABO'\n",
    "    imputation_clean_no_oh['ABO'] = imputation_clean_no_oh['ABO'].str.replace(\"A2\", \"\", case=False, regex=False)\n",
    "    # Replace all instances of \"\" with \"\" in column: 'ABO'\n",
    "    imputation_clean_no_oh.loc[imputation_clean_no_oh['ABO'].str.lower() == \"\".lower(), 'ABO'] = \"A\"\n",
    "    # One-hot encode column: 'ABO'\n",
    "    insert_loc = imputation_clean_no_oh.columns.get_loc('ABO')\n",
    "    imputation_clean_no_oh = pd.concat([imputation_clean_no_oh.iloc[:,:insert_loc], pd.get_dummies(imputation_clean_no_oh.loc[:, ['ABO']]), imputation_clean_no_oh.iloc[:,insert_loc+1:]], axis=1)\n",
    "    # Change column type to float32 for column: 'ETHCAT'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.astype({'ETHCAT': 'float32'})\n",
    "    # Replace all instances of \"U\" with \"0\" in column: 'PERIP_VASC'\n",
    "    imputation_clean_no_oh['PERIP_VASC'] = imputation_clean_no_oh['PERIP_VASC'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"N\" with \"0\" in columns: 'PERIP_VASC', 'DDAVP_DON' and 17 other columns\n",
    "    imputation_clean_no_oh['PERIP_VASC'] = imputation_clean_no_oh['PERIP_VASC'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['DDAVP_DON'] = imputation_clean_no_oh['DDAVP_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['ANTIHYPE_DON'] = imputation_clean_no_oh['ANTIHYPE_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_DIURETICS_DON'] = imputation_clean_no_oh['PT_DIURETICS_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_STEROIDS_DON'] = imputation_clean_no_oh['PT_STEROIDS_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_T3_DON'] = imputation_clean_no_oh['PT_T3_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_T4_DON'] = imputation_clean_no_oh['PT_T4_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['VASODIL_DON'] = imputation_clean_no_oh['VASODIL_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CLIN_INFECT_DON'] = imputation_clean_no_oh['CLIN_INFECT_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_CIG_DON'] = imputation_clean_no_oh['HIST_CIG_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_COCAINE_DON'] = imputation_clean_no_oh['HIST_COCAINE_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_HYPERTENS_DON'] = imputation_clean_no_oh['HIST_HYPERTENS_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_OTH_DRUG_DON'] = imputation_clean_no_oh['HIST_OTH_DRUG_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HEPARIN_DON'] = imputation_clean_no_oh['HEPARIN_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['DIAL_TRR'] = imputation_clean_no_oh['DIAL_TRR'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['MALIG'] = imputation_clean_no_oh['MALIG'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PROTEIN_URINE'] = imputation_clean_no_oh['PROTEIN_URINE'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CARDARREST_NEURO'] = imputation_clean_no_oh['CARDARREST_NEURO'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['INOTROP_SUPPORT_DON'] = imputation_clean_no_oh['INOTROP_SUPPORT_DON'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"1\" in columns: 'PERIP_VASC', 'DDAVP_DON' and 17 other columns\n",
    "    imputation_clean_no_oh['PERIP_VASC'] = imputation_clean_no_oh['PERIP_VASC'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['DDAVP_DON'] = imputation_clean_no_oh['DDAVP_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['ANTIHYPE_DON'] = imputation_clean_no_oh['ANTIHYPE_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_DIURETICS_DON'] = imputation_clean_no_oh['PT_DIURETICS_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_STEROIDS_DON'] = imputation_clean_no_oh['PT_STEROIDS_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_T4_DON'] = imputation_clean_no_oh['PT_T4_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_T3_DON'] = imputation_clean_no_oh['PT_T3_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['VASODIL_DON'] = imputation_clean_no_oh['VASODIL_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CLIN_INFECT_DON'] = imputation_clean_no_oh['CLIN_INFECT_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_CIG_DON'] = imputation_clean_no_oh['HIST_CIG_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_COCAINE_DON'] = imputation_clean_no_oh['HIST_COCAINE_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_HYPERTENS_DON'] = imputation_clean_no_oh['HIST_HYPERTENS_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_OTH_DRUG_DON'] = imputation_clean_no_oh['HIST_OTH_DRUG_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HEPARIN_DON'] = imputation_clean_no_oh['HEPARIN_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['DIAL_TRR'] = imputation_clean_no_oh['DIAL_TRR'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['MALIG'] = imputation_clean_no_oh['MALIG'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PROTEIN_URINE'] = imputation_clean_no_oh['PROTEIN_URINE'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CARDARREST_NEURO'] = imputation_clean_no_oh['CARDARREST_NEURO'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['INOTROP_SUPPORT_DON'] = imputation_clean_no_oh['INOTROP_SUPPORT_DON'].str.replace(\"Y\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"U\" with \"0\" in columns: 'PERIP_VASC', 'DDAVP_DON' and 17 other columns\n",
    "    imputation_clean_no_oh['PERIP_VASC'] = imputation_clean_no_oh['PERIP_VASC'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['DDAVP_DON'] = imputation_clean_no_oh['DDAVP_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['ANTIHYPE_DON'] = imputation_clean_no_oh['ANTIHYPE_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_DIURETICS_DON'] = imputation_clean_no_oh['PT_DIURETICS_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_STEROIDS_DON'] = imputation_clean_no_oh['PT_STEROIDS_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_T3_DON'] = imputation_clean_no_oh['PT_T3_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PT_T4_DON'] = imputation_clean_no_oh['PT_T4_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['VASODIL_DON'] = imputation_clean_no_oh['VASODIL_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CLIN_INFECT_DON'] = imputation_clean_no_oh['CLIN_INFECT_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_CIG_DON'] = imputation_clean_no_oh['HIST_CIG_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_COCAINE_DON'] = imputation_clean_no_oh['HIST_COCAINE_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_HYPERTENS_DON'] = imputation_clean_no_oh['HIST_HYPERTENS_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HIST_OTH_DRUG_DON'] = imputation_clean_no_oh['HIST_OTH_DRUG_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HEPARIN_DON'] = imputation_clean_no_oh['HEPARIN_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['DIAL_TRR'] = imputation_clean_no_oh['DIAL_TRR'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['MALIG'] = imputation_clean_no_oh['MALIG'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['PROTEIN_URINE'] = imputation_clean_no_oh['PROTEIN_URINE'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CARDARREST_NEURO'] = imputation_clean_no_oh['CARDARREST_NEURO'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['INOTROP_SUPPORT_DON'] = imputation_clean_no_oh['INOTROP_SUPPORT_DON'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "   # Replace all instances of \"A1\" with \"A\" in column: 'ABO_DON'\n",
    "    imputation_clean_no_oh['ABO_DON'] = imputation_clean_no_oh['ABO_DON'].str.replace(\"A1\", \"A\", case=False, regex=False)\n",
    "    # Replace all instances of \"A2\" with \"A\" in column: 'ABO_DON'\n",
    "    imputation_clean_no_oh['ABO_DON'] = imputation_clean_no_oh['ABO_DON'].str.replace(\"A2\", \"A\", case=False, regex=False)\n",
    "    # One-hot encode column: 'ABO_DON'\n",
    "    insert_loc = imputation_clean_no_oh.columns.get_loc('ABO_DON')\n",
    "    imputation_clean_no_oh = pd.concat([imputation_clean_no_oh.iloc[:,:insert_loc], pd.get_dummies(imputation_clean_no_oh.loc[:, ['ABO_DON']]), imputation_clean_no_oh.iloc[:,insert_loc+1:]], axis=1)\n",
    "    # Replace all instances of \"M\" with \"0\" in column: 'GENDER_DON'\n",
    "    imputation_clean_no_oh['GENDER_DON'] = imputation_clean_no_oh['GENDER_DON'].str.replace(\"M\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"F\" with \"1\" in column: 'GENDER_DON'\n",
    "    imputation_clean_no_oh['GENDER_DON'] = imputation_clean_no_oh['GENDER_DON'].str.replace(\"F\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"999\" with \"0\" in column: 'ART_RECON'\n",
    "    imputation_clean_no_oh['ART_RECON'] = imputation_clean_no_oh['ART_RECON'].str.replace(\"999\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"3\" with \"0\" in column: 'ART_RECON'\n",
    "    imputation_clean_no_oh['ART_RECON'] = imputation_clean_no_oh['ART_RECON'].str.replace(\"3\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"1\" with \"0\" in column: 'ART_RECON'\n",
    "    imputation_clean_no_oh['ART_RECON'] = imputation_clean_no_oh['ART_RECON'].str.replace(\"1\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"4\" with \"0\" in column: 'ART_RECON'\n",
    "    imputation_clean_no_oh['ART_RECON'] = imputation_clean_no_oh['ART_RECON'].str.replace(\"4\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5\" with \"0\" in column: 'ART_RECON'\n",
    "    imputation_clean_no_oh['ART_RECON'] = imputation_clean_no_oh['ART_RECON'].str.replace(\"5\", \"0\", case=False, regex=False)\n",
    "    # Rename column 'ART_RECON' to 'ART_RECON_is2'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.rename(columns={'ART_RECON': 'ART_RECON_is2'})\n",
    "    # Replace all instances of \"2\" with \"1\" in column: 'ART_RECON_is2'\n",
    "    imputation_clean_no_oh['ART_RECON_is2'] = imputation_clean_no_oh['ART_RECON_is2'].str.replace(\"2\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"4\" with \"999\" in column: 'DUCT_MGMT'\n",
    "    imputation_clean_no_oh['DUCT_MGMT'] = imputation_clean_no_oh['DUCT_MGMT'].str.replace(\"4\", \"999\", case=False, regex=False)\n",
    "    # Replace all instances of \"5\" with \"999\" in column: 'DUCT_MGMT'\n",
    "    imputation_clean_no_oh['DUCT_MGMT'] = imputation_clean_no_oh['DUCT_MGMT'].str.replace(\"5\", \"999\", case=False, regex=False)\n",
    "    # One-hot encode column: 'DUCT_MGMT'\n",
    "    insert_loc = imputation_clean_no_oh.columns.get_loc('DUCT_MGMT')\n",
    "    imputation_clean_no_oh = pd.concat([imputation_clean_no_oh.iloc[:,:insert_loc], pd.get_dummies(imputation_clean_no_oh.loc[:, ['DUCT_MGMT']]), imputation_clean_no_oh.iloc[:,insert_loc+1:]], axis=1)\n",
    "    # Replace all instances of \"2\" with \"0\" in column: 'GRF_PLACEM'\n",
    "    imputation_clean_no_oh['GRF_PLACEM'] = imputation_clean_no_oh['GRF_PLACEM'].str.replace(\"2\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"3\" with \"0\" in column: 'GRF_PLACEM'\n",
    "    imputation_clean_no_oh['GRF_PLACEM'] = imputation_clean_no_oh['GRF_PLACEM'].str.replace(\"3\", \"0\", case=False, regex=False)\n",
    "    # Rename column 'GRF_PLACEM' to 'GRF_PLACEM_is1'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.rename(columns={'GRF_PLACEM': 'GRF_PLACEM_is1'})\n",
    "    # Replace all instances of \"2\" with \"0\" in column: 'VASC_MGMT'\n",
    "    imputation_clean_no_oh['VASC_MGMT'] = imputation_clean_no_oh['VASC_MGMT'].str.replace(\"2\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"3\" with \"0\" in column: 'VASC_MGMT'\n",
    "    imputation_clean_no_oh['VASC_MGMT'] = imputation_clean_no_oh['VASC_MGMT'].str.replace(\"3\", \"0\", case=False, regex=False)\n",
    "    # Rename column 'VASC_MGMT' to 'VASC_MGMT_is1'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.rename(columns={'VASC_MGMT': 'VASC_MGMT_is1'})\n",
    "    # Replace all instances of \"KP\" with \"1\" in column: 'ORGAN'\n",
    "    imputation_clean_no_oh['ORGAN'] = imputation_clean_no_oh['ORGAN'].str.replace(\"KP\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"PA\" with \"0\" in column: 'ORGAN'\n",
    "    imputation_clean_no_oh['ORGAN'] = imputation_clean_no_oh['ORGAN'].str.replace(\"PA\", \"0\", case=False, regex=False)\n",
    "    # Rename column 'ORGAN' to 'ORGAN_isKP'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.rename(columns={'ORGAN': 'ORGAN_isKP'})\n",
    "    # Replace all instances of \"P\" with \"1\" in columns: 'CMV_IGM', 'CMV_IGG' and 5 other columns\n",
    "    imputation_clean_no_oh['CMV_IGM'] = imputation_clean_no_oh['CMV_IGM'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CMV_IGG'] = imputation_clean_no_oh['CMV_IGG'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['EBV_SEROSTATUS'] = imputation_clean_no_oh['EBV_SEROSTATUS'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HBV_CORE'] = imputation_clean_no_oh['HBV_CORE'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HBV_SUR_ANTIGEN'] = imputation_clean_no_oh['HBV_SUR_ANTIGEN'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HCV_SEROSTATUS'] = imputation_clean_no_oh['HCV_SEROSTATUS'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CMV_STATUS'] = imputation_clean_no_oh['CMV_STATUS'].str.replace(\"P\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"N\" with \"0\" in columns: 'CMV_IGG', 'CMV_IGM' and 4 other columns\n",
    "    imputation_clean_no_oh['CMV_IGG'] = imputation_clean_no_oh['CMV_IGG'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CMV_IGM'] = imputation_clean_no_oh['CMV_IGM'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['EBV_SEROSTATUS'] = imputation_clean_no_oh['EBV_SEROSTATUS'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HBV_CORE'] = imputation_clean_no_oh['HBV_CORE'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HBV_SUR_ANTIGEN'] = imputation_clean_no_oh['HBV_SUR_ANTIGEN'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HCV_SEROSTATUS'] = imputation_clean_no_oh['HCV_SEROSTATUS'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"U\" with \"0\" in columns: 'CMV_IGG', 'CMV_IGM' and 5 other columns\n",
    "    imputation_clean_no_oh['CMV_IGG'] = imputation_clean_no_oh['CMV_IGG'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CMV_IGM'] = imputation_clean_no_oh['CMV_IGM'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['EBV_SEROSTATUS'] = imputation_clean_no_oh['EBV_SEROSTATUS'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HBV_CORE'] = imputation_clean_no_oh['HBV_CORE'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HBV_SUR_ANTIGEN'] = imputation_clean_no_oh['HBV_SUR_ANTIGEN'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['HCV_SEROSTATUS'] = imputation_clean_no_oh['HCV_SEROSTATUS'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    imputation_clean_no_oh['CMV_STATUS'] = imputation_clean_no_oh['CMV_STATUS'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"0D\" with \"0\" in column: 'EBV_SEROSTATUS'\n",
    "    imputation_clean_no_oh['EBV_SEROSTATUS'] = imputation_clean_no_oh['EBV_SEROSTATUS'].str.replace(\"0D\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"N\" with \"0\" in column: 'CMV_STATUS'\n",
    "    imputation_clean_no_oh['CMV_STATUS'] = imputation_clean_no_oh['CMV_STATUS'].str.replace(\"N\", \"0\", case=False, regex=False)\n",
    "    # One-hot encode column: 'TX_TYPE'\n",
    "    insert_loc = imputation_clean_no_oh.columns.get_loc('TX_TYPE')\n",
    "    imputation_clean_no_oh = pd.concat([imputation_clean_no_oh.iloc[:,:insert_loc], pd.get_dummies(imputation_clean_no_oh.loc[:, ['TX_TYPE']]), imputation_clean_no_oh.iloc[:,insert_loc+1:]], axis=1)\n",
    "    # Replace all instances of \"U\" with \"0\" in column: 'MED_COND_TRR'\n",
    "    imputation_clean_no_oh['MED_COND_TRR'] = imputation_clean_no_oh['MED_COND_TRR'].str.replace(\"U\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"2\" with \"0\" in column: 'MED_COND_TRR'\n",
    "    imputation_clean_no_oh['MED_COND_TRR'] = imputation_clean_no_oh['MED_COND_TRR'].str.replace(\"2\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"1\" with \"0\" in column: 'MED_COND_TRR'\n",
    "    imputation_clean_no_oh['MED_COND_TRR'] = imputation_clean_no_oh['MED_COND_TRR'].str.replace(\"1\", \"0\", case=False, regex=False)\n",
    "    # Rename column 'MED_COND_TRR' to 'MED_COND_TRR_is3'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.rename(columns={'MED_COND_TRR': 'MED_COND_TRR_is3'})\n",
    "    # Replace all instances of \"3\" with \"1\" in column: 'MED_COND_TRR_is3'\n",
    "    imputation_clean_no_oh['MED_COND_TRR_is3'] = imputation_clean_no_oh['MED_COND_TRR_is3'].str.replace(\"3\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"5001\" with \"1\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5001\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"5002\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5002\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5003\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5003\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5000\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5000\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"999\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"999\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5009\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5009\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5008\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5008\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5005\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5005\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5007\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5007\", \"0\", case=False, regex=False)\n",
    "    # Replace all instances of \"5004\" with \"\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5004\", \"\", case=False, regex=False)\n",
    "    # Replace all instances of \"\" with \"\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh.loc[imputation_clean_no_oh['DIAG_PA'].str.lower() == \"\".lower(), 'DIAG_PA'] = \"0\"\n",
    "    # Replace all instances of \"5006\" with \"0\" in column: 'DIAG_PA'\n",
    "    imputation_clean_no_oh['DIAG_PA'] = imputation_clean_no_oh['DIAG_PA'].str.replace(\"5006\", \"0\", case=False, regex=False)\n",
    "    # Rename column 'DIAG_PA' to 'DIAG_PA_is5001'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.rename(columns={'DIAG_PA': 'DIAG_PA_is5001'})\n",
    "\n",
    "    # Change column type to float32 for columns: 'GENDER', 'ABO_A' and 108 other columns\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.astype({'GENDER': 'float32', 'ABO_A': 'float32', 'ABO_AB': 'float32', 'ABO_B': 'float32', 'ABO_O': 'float32', 'PERIP_VASC': 'float32', 'AGE_DIAB': 'float32', 'CREAT_TRR': 'float32', 'AMIS': 'float32', 'BMIS': 'float32', 'DRMIS': 'float32', 'HLAMIS': 'float32', 'NPKID': 'float32', 'NPPAN': 'float32', 'AGE_DON': 'float32', 'DDAVP_DON': 'float32', 'CMV_DON': 'float32', 'ABO_DON_A': 'float32', 'ABO_DON_AB': 'float32', 'ABO_DON_B': 'float32', 'ABO_DON_O': 'float32', 'GENDER_DON': 'float32', 'NON_HRT_DON': 'float32', 'ANTIHYPE_DON': 'float32', 'BLOOD_INF_CONF_DON': 'float32', 'BUN_DON': 'float32', 'CREAT_DON': 'float32', 'OTHER_INF_CONF_DON': 'float32', 'PT_DIURETICS_DON': 'float32', 'PT_STEROIDS_DON': 'float32', 'PT_T3_DON': 'float32', 'PT_T4_DON': 'float32', 'PULM_INF_CONF_DON': 'float32', 'SGOT_DON': 'float32', 'SGPT_DON': 'float32', 'TBILI_DON': 'float32', 'URINE_INF_CONF_DON': 'float32', 'VASODIL_DON': 'float32', 'CLIN_INFECT_DON': 'float32', 'HIST_CIG_DON': 'float32', 'HIST_COCAINE_DON': 'float32', 'HIST_HYPERTENS_DON': 'float32', 'HIST_CANCER_DON': 'float32', 'HIST_OTH_DRUG_DON': 'float32', 'HEPARIN_DON': 'float32', 'HGT_CM_DON_CALC': 'float32', 'WGT_KG_DON_CALC': 'float32', 'BMI_DON_CALC': 'float32', 'ABO_MAT': 'float32', 'AGE': 'float32', 'DIAL_TRR': 'float32', 'MULTIORG': 'float32', 'ART_RECON_is2': 'float32', 'DUCT_MGMT_1': 'float32', 'DUCT_MGMT_2': 'float32', 'DUCT_MGMT_3': 'float32', 'DUCT_MGMT_999': 'float32', 'GRF_PLACEM_is1': 'float32', 'PA_PRESERV_TM': 'float32', 'VASC_MGMT_is1': 'float32', 'VEN_EXT_GRF': 'float32', 'DIAG_PA_is5001': 'float32', 'DAYSWAIT_CHRON_PA': 'float32', 'TX_PROCEDUR_TY_PA': 'float32', 'ORGAN_isKP': 'float32', 'CMV_IGG': 'float32', 'CMV_IGM': 'float32', 'EBV_SEROSTATUS': 'float32', 'HBV_CORE': 'float32', 'HBV_SUR_ANTIGEN': 'float32', 'HCV_SEROSTATUS': 'float32', 'CMV_STATUS': 'float32', 'TX_TYPE_PAK': 'float32', 'TX_TYPE_PTA': 'float32', 'TX_TYPE_PWK': 'float32', 'TX_TYPE_SKP': 'float32', 'MED_COND_TRR_is3': 'float32', 'MALIG': 'float32', 'HGT_CM_CALC': 'float32', 'WGT_KG_CALC': 'float32', 'BMI_CALC': 'float32', 'PROTEIN_URINE': 'float32', 'CARDARREST_NEURO': 'float32', 'RESUSCIT_DUR': 'float32', 'INOTROP_SUPPORT_DON': 'float32'})\n",
    "\n",
    "    # Replace all instances of \"N\" with \"1\" in column: 'GRF_STAT_PA'\n",
    "    imputation_clean_no_oh['GRF_STAT_PA'] = imputation_clean_no_oh['GRF_STAT_PA'].str.replace(\"N\", \"1\", case=False, regex=False)\n",
    "    # Replace all instances of \"Y\" with \"0\" in column: 'GRF_STAT_PA'\n",
    "    imputation_clean_no_oh['GRF_STAT_PA'] = imputation_clean_no_oh['GRF_STAT_PA'].str.replace(\"Y\", \"0\", case=False, regex=False)\n",
    "    # Change column type to float32 for column: 'GRF_STAT_PA'\n",
    "    imputation_clean_no_oh = imputation_clean_no_oh.astype({'GRF_STAT_PA': 'float32'})\n",
    "    \n",
    "    return imputation_clean_no_oh\n",
    "\n",
    "imputation_clean_no_oh_clean = clean_data(imputation_clean_no_oh.copy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_clean_no_oh_clean[imputation_clean_no_oh_clean.select_dtypes(include=['bool']).columns] = imputation_clean_no_oh_clean.select_dtypes(include=['bool']).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_clean_no_oh_clean = imputation_clean_no_oh_clean.drop(columns=[\"TRR_ID_CODE\", \"ETHCAT\", \"ETHCAT_DON\", \"TX_TYPE_SKP\", \"TX_PROCEDUR_TY_PA\", \"ABO_A\", \"ABO_B\", \"ABO_AB\", \"ABO_O\", \"ABO_DON_A\", \"ABO_DON_B\", \"ABO_DON_AB\", \"ABO_DON_O\", \"TX_TYPE_PAK\", \"DEATH_MECH_DON\", \"DEATH_CIRCUM_DON\", \"COD_CAD_DON\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_clean_no_oh_clean[\"YRS_DIAB\"] = (imputation_clean_no_oh_clean[\"AGE\"] - imputation_clean_no_oh_clean[\"AGE_DIAB\"]) * imputation_clean_no_oh_clean[\"DIAB\"]\n",
    "\n",
    "\n",
    "imputation_clean_no_oh_clean = imputation_clean_no_oh_clean.drop(\"AGE_DIAB\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_clean_no_oh_clean[imputation_clean_no_oh_clean.select_dtypes(include=['int']).columns] = imputation_clean_no_oh_clean.select_dtypes(include=['int']).astype(\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [],
   "source": [
    "skewed_cols = [\"BUN_DON\", \"CREAT_DON\", \"SGOT_DON\", \"SGPT_DON\", \"TBILI_DON\", \"LIPASE\", \"AMYLASE\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_clean_winsor = imputation_clean_no_oh_clean.copy()\n",
    "\n",
    "# Define the lower and upper percentiles\n",
    "lower_percentile = 0.00\n",
    "upper_percentile = 0.95\n",
    "\n",
    "# Apply percentile-based capping\n",
    "imputation_clean_winsor[skewed_cols] = imputation_clean_winsor[skewed_cols].clip(lower=imputation_clean_winsor[skewed_cols].quantile(lower_percentile), upper=imputation_clean_winsor[skewed_cols].quantile(upper_percentile), axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_final = imputation_clean_winsor.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation_final.to_csv(\"data_ready_45.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}