notebooks/04_ESTIMATE_dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Dependencies\" data-toc-modified-id=\"Dependencies-1\">Dependencies</a></span></li><li><span><a href=\"#Functions\" data-toc-modified-id=\"Functions-2\">Functions</a></span></li><li><span><a href=\"#Paths\" data-toc-modified-id=\"Paths-3\">Paths</a></span></li><li><span><a href=\"#Main\" data-toc-modified-id=\"Main-4\">Main</a></span><ul class=\"toc-item\"><li><span><a href=\"#Load-and-cleanup-estimate-files\" data-toc-modified-id=\"Load-and-cleanup-estimate-files-4.1\">Load and cleanup estimate files</a></span><ul class=\"toc-item\"><li><span><a href=\"#GDC\" data-toc-modified-id=\"GDC-4.1.1\">GDC</a></span></li><li><span><a href=\"#DKFZ\" data-toc-modified-id=\"DKFZ-4.1.2\">DKFZ</a></span></li><li><span><a href=\"#CBTTC\" data-toc-modified-id=\"CBTTC-4.1.3\">CBTTC</a></span></li></ul></li><li><span><a href=\"#Create-an-estimate_manifest-file\" data-toc-modified-id=\"Create-an-estimate_manifest-file-4.2\">Create an estimate_manifest file</a></span><ul class=\"toc-item\"><li><span><a href=\"#merge-estimate-and-manifest-of-KF-and-DKFZ-by-sample_iq\" data-toc-modified-id=\"merge-estimate-and-manifest-of-KF-and-DKFZ-by-sample_iq-4.2.1\">merge estimate and manifest of KF and DKFZ by sample_iq</a></span></li><li><span><a href=\"#merge-estimate-and-manifest-of-GDC-by-aliquot_iq\" data-toc-modified-id=\"merge-estimate-and-manifest-of-GDC-by-aliquot_iq-4.2.2\">merge estimate and manifest of GDC by aliquot_iq</a></span></li></ul></li><li><span><a href=\"#Subset-to-primary-tumours\" data-toc-modified-id=\"Subset-to-primary-tumours-4.3\">Subset to primary tumours</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Attaching package: ‘dplyr’\n",
      "\n",
      "\n",
      "The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    filter, lag\n",
      "\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    intersect, setdiff, setequal, union\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "library(dplyr)\n",
    "library(reshape2)\n",
    "library(gtable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions \n",
    "\n",
    "\n",
    "to replace ids with barcodes using the map file (Provided by Komal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2barcodefx <- function(inpath,estimate_outputname){\n",
    "    estimate_output <- read.table(paste0(inpath, estimate_outputname), \n",
    "                              sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)\n",
    "    message(\"Currnet estimate file:\")\n",
    "    print(estimate_outputname)\n",
    "    message(\"dimensions of estimate output file:\")\n",
    "    print(dim(estimate_output))\n",
    "    estimate_output$SampleID <- as.character(estimate_output$SampleID)\n",
    "# remove the X from sampleid\n",
    "    estimate_output$SampleID <- gsub(\"X\", \"\", estimate_output$SampleID)\n",
    "# replace \".\" with \"-\"\n",
    "    estimate_output$SampleID <- gsub(\"[.]\", \"-\", estimate_output$SampleID)\n",
    "    non_existing <- estimate_output[,1][!estimate_output[,1] %in% mappings$sample_id]\n",
    "    non_existing <- as.character(non_existing)\n",
    "    message(\"cannot find this many in mappings\")\n",
    "    print(length(non_existing))\n",
    "    exp_matrix_subset <- estimate_output[!estimate_output[,1] %in% non_existing,]\n",
    "    mappings$barcodes <- as.character(mappings$barcodes)\n",
    "    exp_matrix_subset[,1] <- mappings$barcodes[match(exp_matrix_subset[,1],mappings$sample_id)]\n",
    "    message(\"Dim after id2barcode replacement\")\n",
    "    print(dim(exp_matrix_subset))\n",
    "    return(exp_matrix_subset)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "manifestpath <- \"/Users/anabbi/OneDrive - UHN/Documents/IPD2/Manifests/\"\n",
    "datapath <- \"/Users/anabbi/OneDrive - UHN/Documents/IPD2/Data/\"\n",
    "plotpath <- \"/Users/anabbi/OneDrive - UHN/Documents/IPD2/Plots/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "load(paste0(manifestpath,\"Master_manifest.RData\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " CBTTC   DKFZ TARGET   TCGA \n",
       "   969    199    157   3204 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table(IPD_metadata$group)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and cleanup estimate files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GDC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change TCGA and TARGET ids to barcodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mappings <- read.table(paste0(manifestpath,\"sampleid_barcode_mappings.txt\"),\n",
    "                       sep = \"\\t\", header = T, stringsAsFactors = FALSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mappings$barcodes <- as.character(mappings$sample_barcode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Got the output from pengbo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TARGET_NBL_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 169   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 157   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_COAD_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 352   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 348   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_LUAD_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 598   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 592   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_SKCM_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 470   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 469   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_PRAD_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 556   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 550   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_GBM_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 173   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 172   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_LGG_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 527   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 525   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Currnet estimate file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"TCGA_OV_ESTIMATEoutput.txt\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "dimensions of estimate output file:\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 428   4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "cannot find this many in mappings\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Dim after id2barcode replacement\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 378   4\n"
     ]
    }
   ],
   "source": [
    "NBL_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TARGET_NBL_ESTIMATEoutput.txt\")\n",
    "COAD_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_COAD_ESTIMATEoutput.txt\")\n",
    "LUAD_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_LUAD_ESTIMATEoutput.txt\")\n",
    "SKCM_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_SKCM_ESTIMATEoutput.txt\")\n",
    "PRAD_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_PRAD_ESTIMATEoutput.txt\")\n",
    "GBM_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_GBM_ESTIMATEoutput.txt\")\n",
    "LGG_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_LGG_ESTIMATEoutput.txt\")\n",
    "OV_estimate <- id2barcodefx(paste(datapath,\"ESTIMATE/ESTIMATE_output/\", sep = \"\"),\n",
    "                             \"TCGA_OV_ESTIMATEoutput.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DKFZ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ATRT_estimate <- read.table(paste0(datapath, \"ESTIMATE/ESTIMATE_output/ICGC_ATRTs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt\"),\n",
    "                           sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)\n",
    "\n",
    "ET_estimate <- read.table(paste0(datapath, \"ESTIMATE/ESTIMATE_output/ICGC_ETs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt\"),\n",
    "                           sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)\n",
    "\n",
    "MB_estimate <- read.table(paste0(datapath,\"ESTIMATE/ESTIMATE_output/ICGC_MBs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt\"),\n",
    "                           sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)\n",
    "\n",
    "PA_estimate <- read.table(paste0(datapath,\"ESTIMATE/ESTIMATE_output/ICGC_PAs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt\"),\n",
    "                           sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)\n",
    "\n",
    "pedGBM_estimate <- read.table(paste0(datapath, \"ESTIMATE/ESTIMATE_output/ICGC_GBMs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt\"),\n",
    "                           sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)\n",
    "\n",
    "PDX_estimate <- read.table(paste0(datapath, \"ESTIMATE/ESTIMATE_output/PDX_2017Oct06.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt\"),\n",
    "                           sep = \"\\t\", header = TRUE, stringsAsFactors = FALSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ATRT_estimate$SampleID <- gsub(\"[.]\", \"-\", ATRT_estimate$SampleID)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CBTTC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "CBTTC_estimate <- read.table(paste0(datapath, \"ESTIMATE/ESTIMATE_output/ESTIMATEOutput.CBTTC.txt\"),\n",
    "                              sep = \"\\t\", header = TRUE, check.names = F,\n",
    "                            stringsAsFactors =F, row.names = NULL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>969</li><li>4</li></ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 969\n",
       "\\item 4\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 969\n",
       "2. 4\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] 969   4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dim(CBTTC_estimate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"dataframe\">\n",
       "<caption>A data.frame: 6 × 4</caption>\n",
       "<thead>\n",
       "\t<tr><th></th><th scope=col>row.names</th><th scope=col>StromalScore</th><th scope=col>ImmuneScore</th><th scope=col>ESTIMATEScore</th></tr>\n",
       "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>X7316.2599</td><td> -443.5517</td><td>  638.4058</td><td>  194.8541</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>X7316.1926</td><td> -380.0087</td><td>  657.2281</td><td>  277.2194</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>X7316.489 </td><td> -477.5167</td><td>-1286.7863</td><td>-1764.3030</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>X7316.16  </td><td>  170.9255</td><td>  479.4121</td><td>  650.3375</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>X7316.13  </td><td>  744.6790</td><td> 1353.2365</td><td> 2097.9155</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>X7316.3936</td><td>-1161.8531</td><td> -912.1833</td><td>-2074.0363</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "A data.frame: 6 × 4\n",
       "\\begin{tabular}{r|llll}\n",
       "  & row.names & StromalScore & ImmuneScore & ESTIMATEScore\\\\\n",
       "  & <chr> & <dbl> & <dbl> & <dbl>\\\\\n",
       "\\hline\n",
       "\t1 & X7316.2599 &  -443.5517 &   638.4058 &   194.8541\\\\\n",
       "\t2 & X7316.1926 &  -380.0087 &   657.2281 &   277.2194\\\\\n",
       "\t3 & X7316.489  &  -477.5167 & -1286.7863 & -1764.3030\\\\\n",
       "\t4 & X7316.16   &   170.9255 &   479.4121 &   650.3375\\\\\n",
       "\t5 & X7316.13   &   744.6790 &  1353.2365 &  2097.9155\\\\\n",
       "\t6 & X7316.3936 & -1161.8531 &  -912.1833 & -2074.0363\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "A data.frame: 6 × 4\n",
       "\n",
       "| <!--/--> | row.names &lt;chr&gt; | StromalScore &lt;dbl&gt; | ImmuneScore &lt;dbl&gt; | ESTIMATEScore &lt;dbl&gt; |\n",
       "|---|---|---|---|---|\n",
       "| 1 | X7316.2599 |  -443.5517 |   638.4058 |   194.8541 |\n",
       "| 2 | X7316.1926 |  -380.0087 |   657.2281 |   277.2194 |\n",
       "| 3 | X7316.489  |  -477.5167 | -1286.7863 | -1764.3030 |\n",
       "| 4 | X7316.16   |   170.9255 |   479.4121 |   650.3375 |\n",
       "| 5 | X7316.13   |   744.6790 |  1353.2365 |  2097.9155 |\n",
       "| 6 | X7316.3936 | -1161.8531 |  -912.1833 | -2074.0363 |\n",
       "\n"
      ],
      "text/plain": [
       "  row.names  StromalScore ImmuneScore ESTIMATEScore\n",
       "1 X7316.2599  -443.5517     638.4058    194.8541   \n",
       "2 X7316.1926  -380.0087     657.2281    277.2194   \n",
       "3 X7316.489   -477.5167   -1286.7863  -1764.3030   \n",
       "4 X7316.16     170.9255     479.4121    650.3375   \n",
       "5 X7316.13     744.6790    1353.2365   2097.9155   \n",
       "6 X7316.3936 -1161.8531    -912.1833  -2074.0363   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "head(CBTTC_estimate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames(CBTTC_estimate)[colnames(CBTTC_estimate) == \"row.names\"] <- \"SampleID\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "CBTTC_estimate$SampleID <- gsub(\"X\", \"\",CBTTC_estimate$SampleID)\n",
    "CBTTC_estimate$SampleID <- gsub(\"7316.\", \"7316-\",CBTTC_estimate$SampleID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"dataframe\">\n",
       "<caption>A data.frame: 6 × 4</caption>\n",
       "<thead>\n",
       "\t<tr><th></th><th scope=col>SampleID</th><th scope=col>StromalScore</th><th scope=col>ImmuneScore</th><th scope=col>ESTIMATEScore</th></tr>\n",
       "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>7316-2599</td><td> -443.5517</td><td>  638.4058</td><td>  194.8541</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>7316-1926</td><td> -380.0087</td><td>  657.2281</td><td>  277.2194</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>7316-489 </td><td> -477.5167</td><td>-1286.7863</td><td>-1764.3030</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>7316-16  </td><td>  170.9255</td><td>  479.4121</td><td>  650.3375</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>7316-13  </td><td>  744.6790</td><td> 1353.2365</td><td> 2097.9155</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>7316-3936</td><td>-1161.8531</td><td> -912.1833</td><td>-2074.0363</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "A data.frame: 6 × 4\n",
       "\\begin{tabular}{r|llll}\n",
       "  & SampleID & StromalScore & ImmuneScore & ESTIMATEScore\\\\\n",
       "  & <chr> & <dbl> & <dbl> & <dbl>\\\\\n",
       "\\hline\n",
       "\t1 & 7316-2599 &  -443.5517 &   638.4058 &   194.8541\\\\\n",
       "\t2 & 7316-1926 &  -380.0087 &   657.2281 &   277.2194\\\\\n",
       "\t3 & 7316-489  &  -477.5167 & -1286.7863 & -1764.3030\\\\\n",
       "\t4 & 7316-16   &   170.9255 &   479.4121 &   650.3375\\\\\n",
       "\t5 & 7316-13   &   744.6790 &  1353.2365 &  2097.9155\\\\\n",
       "\t6 & 7316-3936 & -1161.8531 &  -912.1833 & -2074.0363\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "A data.frame: 6 × 4\n",
       "\n",
       "| <!--/--> | SampleID &lt;chr&gt; | StromalScore &lt;dbl&gt; | ImmuneScore &lt;dbl&gt; | ESTIMATEScore &lt;dbl&gt; |\n",
       "|---|---|---|---|---|\n",
       "| 1 | 7316-2599 |  -443.5517 |   638.4058 |   194.8541 |\n",
       "| 2 | 7316-1926 |  -380.0087 |   657.2281 |   277.2194 |\n",
       "| 3 | 7316-489  |  -477.5167 | -1286.7863 | -1764.3030 |\n",
       "| 4 | 7316-16   |   170.9255 |   479.4121 |   650.3375 |\n",
       "| 5 | 7316-13   |   744.6790 |  1353.2365 |  2097.9155 |\n",
       "| 6 | 7316-3936 | -1161.8531 |  -912.1833 | -2074.0363 |\n",
       "\n"
      ],
      "text/plain": [
       "  SampleID  StromalScore ImmuneScore ESTIMATEScore\n",
       "1 7316-2599  -443.5517     638.4058    194.8541   \n",
       "2 7316-1926  -380.0087     657.2281    277.2194   \n",
       "3 7316-489   -477.5167   -1286.7863  -1764.3030   \n",
       "4 7316-16     170.9255     479.4121    650.3375   \n",
       "5 7316-13     744.6790    1353.2365   2097.9155   \n",
       "6 7316-3936 -1161.8531    -912.1833  -2074.0363   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "head(CBTTC_estimate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an estimate_manifest file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "KF_DKFZ_estimate <- rbind(PA_estimate, #DKFZ\n",
    "                        ET_estimate,\n",
    "                        pedGBM_estimate,\n",
    "                        ATRT_estimate,\n",
    "                        MB_estimate,\n",
    "                        CBTTC_estimate )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge estimate and manifest of KF and DKFZ by sample_iq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames(KF_DKFZ_estimate)[colnames(KF_DKFZ_estimate) == \"SampleID\"] <- \"sample_id\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "KF_DKFZ_estimate_manifest <- merge(KF_DKFZ_estimate, IPD_metadata, by = \"sample_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "CBTTC  DKFZ \n",
       "  969   199 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table(KF_DKFZ_estimate_manifest$group)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge estimate and manifest of GDC by aliquot_iq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "GDC_estimate <- rbind(NBL_estimate,\n",
    "                     OV_estimate,\n",
    "                     PRAD_estimate,\n",
    "                     COAD_estimate,\n",
    "                     SKCM_estimate,\n",
    "                     GBM_estimate,\n",
    "                     LGG_estimate,\n",
    "                     LUAD_estimate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames(GDC_estimate)[colnames(GDC_estimate) == \"SampleID\"] <- \"aliquot_id\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "GDC_estimate_manifest <- merge(GDC_estimate, IPD_metadata, by = \"aliquot_id\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "17 TCGA samples missing from IPD_metadata, because their raw files were not available through cavatica, but their tpm file was forwarded to me by komal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_estimate_manifest <- rbind(GDC_estimate_manifest,\n",
    "                                KF_DKFZ_estimate_manifest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subset to primary tumours \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimate_manifest_primary <- total_estimate_manifest[grepl(\"Initial\",total_estimate_manifest$sample_type) |\n",
    "                                                     grepl(\"Primary\", total_estimate_manifest$sample_type), ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimate_manifest_primary <- rbind(estimate_manifest_primary,\n",
    "                                  total_estimate_manifest[total_estimate_manifest$group == \"KF_NBL\",])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>3527</li><li>21</li></ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 3527\n",
       "\\item 21\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 3527\n",
       "2. 21\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] 3527   21"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dim(estimate_manifest_primary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " CBTTC   DKFZ TARGET   TCGA \n",
       "   969    199    157   3017 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table(total_estimate_manifest$group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " CBTTC   DKFZ TARGET   TCGA \n",
       "   730    196    149   2452 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table(estimate_manifest_primary$group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(estimate_manifest_primary,\n",
    "     file = paste0(datapath, \"ESTIMATE/estimate_manifest_primary.RData\"))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(PDX_estimate,\n",
    "     file = paste0(datapath, \"ESTIMATE/PDX_estimate.RData\"))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R 4.0",
   "language": "R",
   "name": "ir40"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "4.0.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "286.594px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}