src/preprocess_pancancer_TCGA_normalized_RNA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "81029769",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 39.0M  100 39.0M    0     0  9132k      0  0:00:04  0:00:04 --:--:-- 9134k\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 46129  100 46129    0     0   280k      0 --:--:-- --:--:-- --:--:--  281k\n"
     ]
    }
   ],
   "source": [
    "# Download Pancancer-normalized from Xena database:  \n",
    "\n",
    "# Example below with TCGA LUAD cohort\n",
    "!mkdir -p ./data\n",
    "!curl -o ./data/HiSeqV2_PANCAN.gz https://tcga-xena-hub.s3.us-east-1.amazonaws.com/download/TCGA.LUAD.sampleMap%2FHiSeqV2_PANCAN.gz\n",
    "!gunzip ./data/HiSeqV2_PANCAN.gz\n",
    "\n",
    "# Download Hallmark genes \n",
    "!curl -L -o ./data/hallmarks_signatures.csv https://raw.githubusercontent.com/mahmoodlab/SurvPath/main/datasets_csv/metadata/hallmarks_signatures.csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "5e3909e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import os \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "7e44be4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read pan-cancer normalized RNA data \n",
    "df = pd.read_csv('../data/HiSeqV2_PANCAN', delimiter='\\t')\n",
    "\n",
    "# read \n",
    "hallmarks = pd.read_csv('./data_csvs/rna/metadata/hallmarks_signatures.csv').values.flatten()\n",
    "hallmarks = hallmarks[~pd.isnull(hallmarks)]\n",
    "hallmarks = pd.unique(hallmarks)\n",
    "\n",
    "# filter out and transpose\n",
    "df = df[df['sample'].isin(hallmarks)].reset_index(drop=True)\n",
    "df = df.set_index('sample')\n",
    "df = df.transpose()\n",
    "df = df.sort_index(axis=1)\n",
    "df = df.reset_index()\n",
    "df = df.rename(columns={'index': 'sample'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "15b00353",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>sample</th>\n",
       "      <th>sample</th>\n",
       "      <th>A2M</th>\n",
       "      <th>AAAS</th>\n",
       "      <th>AADAT</th>\n",
       "      <th>ABAT</th>\n",
       "      <th>ABCA1</th>\n",
       "      <th>ABCA2</th>\n",
       "      <th>ABCA3</th>\n",
       "      <th>ABCA4</th>\n",
       "      <th>ABCA5</th>\n",
       "      <th>...</th>\n",
       "      <th>ZNF292</th>\n",
       "      <th>ZNF365</th>\n",
       "      <th>ZNF639</th>\n",
       "      <th>ZNF707</th>\n",
       "      <th>ZNFX1</th>\n",
       "      <th>ZNRF4</th>\n",
       "      <th>ZPBP</th>\n",
       "      <th>ZW10</th>\n",
       "      <th>ZWINT</th>\n",
       "      <th>ZYX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-69-7978-01</td>\n",
       "      <td>1.251923</td>\n",
       "      <td>-0.529402</td>\n",
       "      <td>-0.042641</td>\n",
       "      <td>-0.630948</td>\n",
       "      <td>1.379942</td>\n",
       "      <td>0.748904</td>\n",
       "      <td>1.56122</td>\n",
       "      <td>1.146719</td>\n",
       "      <td>-0.770085</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.125712</td>\n",
       "      <td>-0.552304</td>\n",
       "      <td>0.053797</td>\n",
       "      <td>-0.168966</td>\n",
       "      <td>1.330985</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.145589</td>\n",
       "      <td>0.227668</td>\n",
       "      <td>1.081245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-62-8399-01</td>\n",
       "      <td>2.320823</td>\n",
       "      <td>-0.246702</td>\n",
       "      <td>-1.047241</td>\n",
       "      <td>-3.088048</td>\n",
       "      <td>0.775042</td>\n",
       "      <td>0.585904</td>\n",
       "      <td>2.82752</td>\n",
       "      <td>1.113519</td>\n",
       "      <td>0.130115</td>\n",
       "      <td>...</td>\n",
       "      <td>0.464988</td>\n",
       "      <td>-0.118404</td>\n",
       "      <td>-0.382503</td>\n",
       "      <td>-0.179466</td>\n",
       "      <td>0.186785</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.615389</td>\n",
       "      <td>0.881668</td>\n",
       "      <td>0.135045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TCGA-78-7539-01</td>\n",
       "      <td>0.904123</td>\n",
       "      <td>-0.288402</td>\n",
       "      <td>0.149659</td>\n",
       "      <td>-1.503948</td>\n",
       "      <td>-0.165658</td>\n",
       "      <td>-0.706396</td>\n",
       "      <td>3.09332</td>\n",
       "      <td>2.268119</td>\n",
       "      <td>0.792215</td>\n",
       "      <td>...</td>\n",
       "      <td>0.851588</td>\n",
       "      <td>-0.578704</td>\n",
       "      <td>-1.047503</td>\n",
       "      <td>-0.399266</td>\n",
       "      <td>0.492385</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.144389</td>\n",
       "      <td>-0.520132</td>\n",
       "      <td>-0.429255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TCGA-50-5931-11</td>\n",
       "      <td>2.681523</td>\n",
       "      <td>-0.972402</td>\n",
       "      <td>0.333659</td>\n",
       "      <td>-1.544748</td>\n",
       "      <td>1.744042</td>\n",
       "      <td>-1.117096</td>\n",
       "      <td>4.67902</td>\n",
       "      <td>-0.594981</td>\n",
       "      <td>-0.330485</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.727612</td>\n",
       "      <td>1.993296</td>\n",
       "      <td>0.129797</td>\n",
       "      <td>-0.773966</td>\n",
       "      <td>0.857585</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>0.653792</td>\n",
       "      <td>-0.568911</td>\n",
       "      <td>-2.627832</td>\n",
       "      <td>0.440545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TCGA-73-4658-01</td>\n",
       "      <td>2.265523</td>\n",
       "      <td>-1.073002</td>\n",
       "      <td>-1.748941</td>\n",
       "      <td>-1.465748</td>\n",
       "      <td>0.553142</td>\n",
       "      <td>-1.016396</td>\n",
       "      <td>2.81752</td>\n",
       "      <td>0.108319</td>\n",
       "      <td>-2.572885</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.849512</td>\n",
       "      <td>-0.091004</td>\n",
       "      <td>0.035497</td>\n",
       "      <td>-0.955166</td>\n",
       "      <td>0.685385</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.008289</td>\n",
       "      <td>-0.391132</td>\n",
       "      <td>0.822545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>571</th>\n",
       "      <td>TCGA-55-7727-01</td>\n",
       "      <td>1.028323</td>\n",
       "      <td>-1.278202</td>\n",
       "      <td>1.057759</td>\n",
       "      <td>-0.080448</td>\n",
       "      <td>1.168042</td>\n",
       "      <td>-2.314796</td>\n",
       "      <td>2.78952</td>\n",
       "      <td>3.842119</td>\n",
       "      <td>0.257515</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449088</td>\n",
       "      <td>-0.256104</td>\n",
       "      <td>-0.013903</td>\n",
       "      <td>-0.964266</td>\n",
       "      <td>0.631485</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.196389</td>\n",
       "      <td>1.326768</td>\n",
       "      <td>-2.454855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>TCGA-91-6831-01</td>\n",
       "      <td>0.180323</td>\n",
       "      <td>-0.074402</td>\n",
       "      <td>0.131459</td>\n",
       "      <td>-1.874248</td>\n",
       "      <td>0.120942</td>\n",
       "      <td>-1.629596</td>\n",
       "      <td>0.04442</td>\n",
       "      <td>0.134219</td>\n",
       "      <td>-1.186185</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.219912</td>\n",
       "      <td>1.259796</td>\n",
       "      <td>0.308797</td>\n",
       "      <td>-0.111366</td>\n",
       "      <td>-0.377015</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.909589</td>\n",
       "      <td>1.985368</td>\n",
       "      <td>-0.602655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>TCGA-MN-A4N4-01</td>\n",
       "      <td>0.766923</td>\n",
       "      <td>-0.153902</td>\n",
       "      <td>0.806759</td>\n",
       "      <td>0.344752</td>\n",
       "      <td>0.234242</td>\n",
       "      <td>-0.002596</td>\n",
       "      <td>-0.73318</td>\n",
       "      <td>2.934519</td>\n",
       "      <td>-1.610385</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.087312</td>\n",
       "      <td>0.887996</td>\n",
       "      <td>-0.166103</td>\n",
       "      <td>0.690434</td>\n",
       "      <td>-0.127415</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>0.150689</td>\n",
       "      <td>0.049868</td>\n",
       "      <td>0.188645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574</th>\n",
       "      <td>TCGA-55-8302-01</td>\n",
       "      <td>0.393523</td>\n",
       "      <td>0.145198</td>\n",
       "      <td>-0.341541</td>\n",
       "      <td>-2.544348</td>\n",
       "      <td>0.342342</td>\n",
       "      <td>-0.726296</td>\n",
       "      <td>2.40002</td>\n",
       "      <td>2.387019</td>\n",
       "      <td>-1.356085</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.499912</td>\n",
       "      <td>2.543696</td>\n",
       "      <td>-0.886603</td>\n",
       "      <td>0.936634</td>\n",
       "      <td>1.433085</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>-0.760111</td>\n",
       "      <td>1.378868</td>\n",
       "      <td>0.246945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>TCGA-MP-A4TK-01</td>\n",
       "      <td>1.020823</td>\n",
       "      <td>-0.097502</td>\n",
       "      <td>-0.198641</td>\n",
       "      <td>-1.482548</td>\n",
       "      <td>-0.169758</td>\n",
       "      <td>0.314804</td>\n",
       "      <td>0.96132</td>\n",
       "      <td>3.816719</td>\n",
       "      <td>-1.683785</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033888</td>\n",
       "      <td>1.835396</td>\n",
       "      <td>-0.790603</td>\n",
       "      <td>0.584634</td>\n",
       "      <td>-0.206515</td>\n",
       "      <td>-0.103518</td>\n",
       "      <td>-0.168308</td>\n",
       "      <td>-0.222211</td>\n",
       "      <td>0.185368</td>\n",
       "      <td>1.001645</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>576 rows × 4169 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "sample           sample       A2M      AAAS     AADAT      ABAT     ABCA1  \\\n",
       "0       TCGA-69-7978-01  1.251923 -0.529402 -0.042641 -0.630948  1.379942   \n",
       "1       TCGA-62-8399-01  2.320823 -0.246702 -1.047241 -3.088048  0.775042   \n",
       "2       TCGA-78-7539-01  0.904123 -0.288402  0.149659 -1.503948 -0.165658   \n",
       "3       TCGA-50-5931-11  2.681523 -0.972402  0.333659 -1.544748  1.744042   \n",
       "4       TCGA-73-4658-01  2.265523 -1.073002 -1.748941 -1.465748  0.553142   \n",
       "..                  ...       ...       ...       ...       ...       ...   \n",
       "571     TCGA-55-7727-01  1.028323 -1.278202  1.057759 -0.080448  1.168042   \n",
       "572     TCGA-91-6831-01  0.180323 -0.074402  0.131459 -1.874248  0.120942   \n",
       "573     TCGA-MN-A4N4-01  0.766923 -0.153902  0.806759  0.344752  0.234242   \n",
       "574     TCGA-55-8302-01  0.393523  0.145198 -0.341541 -2.544348  0.342342   \n",
       "575     TCGA-MP-A4TK-01  1.020823 -0.097502 -0.198641 -1.482548 -0.169758   \n",
       "\n",
       "sample     ABCA2    ABCA3     ABCA4     ABCA5  ...    ZNF292    ZNF365  \\\n",
       "0       0.748904  1.56122  1.146719 -0.770085  ... -0.125712 -0.552304   \n",
       "1       0.585904  2.82752  1.113519  0.130115  ...  0.464988 -0.118404   \n",
       "2      -0.706396  3.09332  2.268119  0.792215  ...  0.851588 -0.578704   \n",
       "3      -1.117096  4.67902 -0.594981 -0.330485  ... -0.727612  1.993296   \n",
       "4      -1.016396  2.81752  0.108319 -2.572885  ... -0.849512 -0.091004   \n",
       "..           ...      ...       ...       ...  ...       ...       ...   \n",
       "571    -2.314796  2.78952  3.842119  0.257515  ...  0.449088 -0.256104   \n",
       "572    -1.629596  0.04442  0.134219 -1.186185  ... -0.219912  1.259796   \n",
       "573    -0.002596 -0.73318  2.934519 -1.610385  ... -0.087312  0.887996   \n",
       "574    -0.726296  2.40002  2.387019 -1.356085  ... -0.499912  2.543696   \n",
       "575     0.314804  0.96132  3.816719 -1.683785  ...  0.033888  1.835396   \n",
       "\n",
       "sample    ZNF639    ZNF707     ZNFX1     ZNRF4      ZPBP      ZW10     ZWINT  \\\n",
       "0       0.053797 -0.168966  1.330985 -0.103518 -0.168308  0.145589  0.227668   \n",
       "1      -0.382503 -0.179466  0.186785 -0.103518 -0.168308  0.615389  0.881668   \n",
       "2      -1.047503 -0.399266  0.492385 -0.103518 -0.168308  0.144389 -0.520132   \n",
       "3       0.129797 -0.773966  0.857585 -0.103518  0.653792 -0.568911 -2.627832   \n",
       "4       0.035497 -0.955166  0.685385 -0.103518 -0.168308  0.008289 -0.391132   \n",
       "..           ...       ...       ...       ...       ...       ...       ...   \n",
       "571    -0.013903 -0.964266  0.631485 -0.103518 -0.168308  0.196389  1.326768   \n",
       "572     0.308797 -0.111366 -0.377015 -0.103518 -0.168308  0.909589  1.985368   \n",
       "573    -0.166103  0.690434 -0.127415 -0.103518 -0.168308  0.150689  0.049868   \n",
       "574    -0.886603  0.936634  1.433085 -0.103518 -0.168308 -0.760111  1.378868   \n",
       "575    -0.790603  0.584634 -0.206515 -0.103518 -0.168308 -0.222211  0.185368   \n",
       "\n",
       "sample       ZYX  \n",
       "0       1.081245  \n",
       "1       0.135045  \n",
       "2      -0.429255  \n",
       "3       0.440545  \n",
       "4       0.822545  \n",
       "..           ...  \n",
       "571    -2.454855  \n",
       "572    -0.602655  \n",
       "573     0.188645  \n",
       "574     0.246945  \n",
       "575     1.001645  \n",
       "\n",
       "[576 rows x 4169 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv('../luad_hallmarks_rna_clean.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hest",
   "language": "python",
   "name": "hest"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}