munge/045_check-depmap-modeling-data_original.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7fe91a96",
   "metadata": {},
   "source": [
    "# Check of modeling data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0179a9ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:06.366026Z",
     "iopub.status.busy": "2021-10-04T22:10:06.365330Z",
     "iopub.status.idle": "2021-10-04T22:10:14.580685Z",
     "shell.execute_reply": "2021-10-04T22:10:14.581175Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import dask.dataframe as dd\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from dask.distributed import Client, progress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb59d032",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:14.586774Z",
     "iopub.status.busy": "2021-10-04T22:10:14.585987Z",
     "iopub.status.idle": "2021-10-04T22:10:18.215682Z",
     "shell.execute_reply": "2021-10-04T22:10:18.215144Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "client = Client(n_workers=4, threads_per_worker=2, memory_limit=\"32GB\")\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e312422f-c936-465c-b716-21f37c6b41cd",
   "metadata": {},
   "source": [
    "Papermill parameters:\n",
    "\n",
    "`depmap_modeling_df`: The path to the full DepMap modeling data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2345a8aa-8fd2-4dd6-902f-ac1845e40b65",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "DEPMAP_MODELING_DF: str = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f4ed4a7-6083-410c-8737-129ce1c9372b",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert DEPMAP_MODELING_DF != \"\", \"No path provided for the modeling data.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b723ad3c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:18.221335Z",
     "iopub.status.busy": "2021-10-04T22:10:18.220604Z",
     "iopub.status.idle": "2021-10-04T22:10:18.224606Z",
     "shell.execute_reply": "2021-10-04T22:10:18.224102Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "depmap_modeling_df_path = Path(DEPMAP_MODELING_DF)\n",
    "\n",
    "if not depmap_modeling_df_path.exists():\n",
    "    raise FileNotFoundError(f\"Could not find '{str(depmap_modeling_df_path)}'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "717b22fc-b2af-495c-b03f-aa9720fe0120",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:18.230780Z",
     "iopub.status.busy": "2021-10-04T22:10:18.230180Z",
     "iopub.status.idle": "2021-10-04T22:10:18.319460Z",
     "shell.execute_reply": "2021-10-04T22:10:18.319937Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.read_csv(depmap_modeling_df_path, low_memory=False, nrows=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe240c50",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:18.348969Z",
     "iopub.status.busy": "2021-10-04T22:10:18.328352Z",
     "iopub.status.idle": "2021-10-04T22:10:18.381121Z",
     "shell.execute_reply": "2021-10-04T22:10:18.380598Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "depmap_modeling_df = dd.read_csv(\n",
    "    depmap_modeling_df_path,\n",
    "    dtype={\n",
    "        \"age\": \"float64\",\n",
    "        \"p_dna_batch\": \"object\",\n",
    "        \"primary_or_metastasis\": \"object\",\n",
    "        \"counts_final\": \"float64\",\n",
    "        \"counts_initial\": \"float64\",\n",
    "    },\n",
    "    low_memory=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f56f5af3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:18.433363Z",
     "iopub.status.busy": "2021-10-04T22:10:18.395652Z",
     "iopub.status.idle": "2021-10-04T22:10:21.005740Z",
     "shell.execute_reply": "2021-10-04T22:10:21.006214Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "depmap_modeling_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a321087e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:21.013409Z",
     "iopub.status.busy": "2021-10-04T22:10:21.011925Z",
     "iopub.status.idle": "2021-10-04T22:10:21.017362Z",
     "shell.execute_reply": "2021-10-04T22:10:21.016546Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "depmap_modeling_df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8f07fc3",
   "metadata": {},
   "source": [
    "## Basic checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef9977a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:21.021965Z",
     "iopub.status.busy": "2021-10-04T22:10:21.021390Z",
     "iopub.status.idle": "2021-10-04T22:10:21.025101Z",
     "shell.execute_reply": "2021-10-04T22:10:21.025580Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "FAILED_CHECKS = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c9001e4-2bd2-44f8-a87b-a6133a26c5fe",
   "metadata": {},
   "source": [
    "Check that specific columns exist (prevents some really bonehead discoveries later on...)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4bab8e1-e575-43f7-adab-4d7a39eb04a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:21.032300Z",
     "iopub.status.busy": "2021-10-04T22:10:21.031652Z",
     "iopub.status.idle": "2021-10-04T22:10:21.034083Z",
     "shell.execute_reply": "2021-10-04T22:10:21.034611Z"
    }
   },
   "outputs": [],
   "source": [
    "cols_that_should_exist = [\n",
    "    \"depmap_id\",\n",
    "    \"sgrna\",\n",
    "    \"hugo_symbol\",\n",
    "    \"lfc\",\n",
    "    \"screen\",\n",
    "    \"num_mutations\",\n",
    "    \"is_mutated\",\n",
    "    \"lineage\",\n",
    "    \"counts_final\",\n",
    "    \"p_dna_batch\",\n",
    "    \"primary_or_metastasis\",\n",
    "]\n",
    "\n",
    "missing_cols = [\n",
    "    col for col in cols_that_should_exist if col not in depmap_modeling_df.columns\n",
    "]\n",
    "if len(missing_cols) != 0:\n",
    "    print(f\"Some columns ({len(missing_cols)}) that should be present are not 😦\")\n",
    "    print(\"  missing columns: \" + \", \".join(missing_cols))\n",
    "    FAILED_CHECKS += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a944c3e",
   "metadata": {},
   "source": [
    "Check that specific columns have no missing (`NA`) values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37cecb68",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:10:21.047491Z",
     "iopub.status.busy": "2021-10-04T22:10:21.042899Z",
     "iopub.status.idle": "2021-10-04T22:13:29.451180Z",
     "shell.execute_reply": "2021-10-04T22:13:29.451757Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cols_without_na = [\n",
    "    \"depmap_id\",\n",
    "    \"sgrna\",\n",
    "    \"hugo_symbol\",\n",
    "    \"lfc\",\n",
    "    \"screen\",\n",
    "    \"num_mutations\",\n",
    "    \"is_mutated\",\n",
    "    \"lineage\",\n",
    "]\n",
    "\n",
    "na_checks = depmap_modeling_df.isna()[cols_without_na].any().compute()\n",
    "num_missed_checks = na_checks.sum()\n",
    "\n",
    "if num_missed_checks > 0:\n",
    "    FAILED_CHECKS += num_missed_checks\n",
    "    print(na_checks[na_checks])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d029d43",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:13:29.461404Z",
     "iopub.status.busy": "2021-10-04T22:13:29.460460Z",
     "iopub.status.idle": "2021-10-04T22:13:29.463984Z",
     "shell.execute_reply": "2021-10-04T22:13:29.463375Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "na_checks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9da17612",
   "metadata": {},
   "source": [
    "Check that all combinations of cell line, sgRNA, and experimental replicate only appear once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e554ea19",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:13:29.558558Z",
     "iopub.status.busy": "2021-10-04T22:13:29.501861Z",
     "iopub.status.idle": "2021-10-04T22:21:27.920422Z",
     "shell.execute_reply": "2021-10-04T22:21:27.920926Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "grp_cols = [\"depmap_id\", \"sgrna\", \"replicate_id\"]\n",
    "ct_df = (\n",
    "    depmap_modeling_df.assign(n=1)[grp_cols + [\"n\"]]\n",
    "    .groupby(grp_cols)\n",
    "    .count()\n",
    "    .query(\"n > 1\")\n",
    "    .compute()\n",
    ")\n",
    "\n",
    "if not ct_df.shape[0] == 0:\n",
    "    print(\"There are some sgRNA with multiple targets.\")\n",
    "    print(ct_df.head(20))\n",
    "    FAILED_CHECKS += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3671ee6b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:21:27.927269Z",
     "iopub.status.busy": "2021-10-04T22:21:27.926654Z",
     "iopub.status.idle": "2021-10-04T22:21:27.929601Z",
     "shell.execute_reply": "2021-10-04T22:21:27.928987Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if FAILED_CHECKS > 0:\n",
    "    raise Exception(f\"There were {FAILED_CHECKS} failed checks.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a80ef0fa",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14a9d255",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-04T22:21:27.936074Z",
     "iopub.status.busy": "2021-10-04T22:21:27.935413Z",
     "iopub.status.idle": "2021-10-04T22:21:28.237858Z",
     "shell.execute_reply": "2021-10-04T22:21:28.237334Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext watermark\n",
    "%watermark -d -u -v -iv -b -h -m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bd640d9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}