Genie DAG import scripts

import-scripts/genie-airflow-clone-db.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Script for running pre-import steps
+# Consists of the following:
+# - Determine which database is "production" vs "not production"
+# - Drop tables in the non-production database
+# - Clone the production database into the non-production database
+
+PORTAL_SCRIPTS_DIRECTORY=$1
+if [ -z $PORTAL_SCRIPTS_DIRECTORY ]; then
+    PORTAL_SCRIPTS_DIRECTORY="/data/portal-cron/scripts"
+fi
+AUTOMATION_ENV_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/automation-environment.sh"
+if [ ! -f $AUTOMATION_ENV_SCRIPT_FILEPATH ] ; then
+    echo "`date`: Unable to locate $AUTOMATION_ENV_SCRIPT_FILEPATH, exiting..."
+    exit 1
+fi
+source $AUTOMATION_ENV_SCRIPT_FILEPATH
+
+# Create tmp directory for processing
+tmp=$PORTAL_HOME/tmp/import-cron-genie
+if ! [ -d "$tmp" ] ; then
+    if ! mkdir -p "$tmp" ; then
+        echo "Error : could not create tmp directory '$tmp'" >&2
+        exit 1
+    fi
+fi
+if [[ -d "$tmp" && "$tmp" != "/" ]]; then
+    rm -rf "$tmp"/*
+fi
+
+MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/pipelines-credentials/manage_genie_database_update_tools.properties"
+SET_UPDATE_PROCESS_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/set_update_process_state.sh"
+GET_DB_IN_PROD_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/get_database_currently_in_production.sh"
+DROP_TABLES_FROM_MYSQL_DATABASE_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/drop_tables_in_mysql_database.sh"
+CLONE_MYSQL_DATABASE_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/clone_mysql_database.sh"
+
+# Update the process status database
+if ! $SET_UPDATE_PROCESS_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH running ; then
+    echo "Error during execution of $SET_UPDATE_PROCESS_SCRIPT_FILEPATH : could not set running state" >&2
+    exit 1
+fi
+
+# Get the current production database color
+current_production_database_color=$(sh $GET_DB_IN_PROD_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH)
+source_database_color="unset"
+destination_database_color="unset"
+if [ ${current_production_database_color:0:4} == "blue" ] ; then
+    source_database_color="blue"
+    destination_database_color="green"
+fi
+if [ ${current_production_database_color:0:5} == "green" ] ; then
+    source_database_color="green"
+    destination_database_color="blue"
+fi
+if [ "$destination_database_color" == "unset" ] ; then
+    echo "Error during determination of the destination database color" >&2
+    exit 1
+fi
+
+echo "we are going to clone $source_database_color into $destination_database_color and then import into $destination_database_color"
+# Drop tables in the non-production database to make space for cloning
+if ! $DROP_TABLES_FROM_MYSQL_DATABASE_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH $destination_database_color ; then
+    message="Error during dropping of tables from mysql database $destination_database_color"
+    echo $message >&2
+    exit 1
+else
+    # Clone the content of the production MySQL database into the non-production database
+    if ! $CLONE_MYSQL_DATABASE_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH $source_database_color $destination_database_color ; then
+        message="Error during cloning the mysql database (from $source_database_color to $destination_database_color)"
+        echo $message >&2
+        exit 1
+    fi
+fi

import-scripts/genie-airflow-import-clickhouse.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# Script for updating ClickHouse DB
+# Consists of the following:
+# - Drop ClickHouse tables
+# - Copy MySQL tables to ClickHouse
+# - Create derived ClickHouse tables
+
+PORTAL_SCRIPTS_DIRECTORY=$1
+if [ -z $PORTAL_SCRIPTS_DIRECTORY ]; then
+    PORTAL_SCRIPTS_DIRECTORY="/data/portal-cron/scripts"
+fi
+AUTOMATION_ENV_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/automation-environment.sh"
+if [ ! -f $AUTOMATION_ENV_SCRIPT_FILEPATH ] ; then
+    echo "`date`: Unable to locate $AUTOMATION_ENV_SCRIPT_FILEPATH, exiting..."
+    exit 1
+fi
+source $AUTOMATION_ENV_SCRIPT_FILEPATH
+
+MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/pipelines-credentials/manage_genie_database_update_tools.properties"
+GET_DB_IN_PROD_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/get_database_currently_in_production.sh"
+DROP_TABLES_FROM_CLICKHOUSE_DATABASE_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/drop_tables_in_clickhouse_database.sh"
+COPY_TABLES_FROM_MYSQL_TO_CLICKHOUSE_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/copy_mysql_database_tables_to_clickhouse.sh"
+DOWNLOAD_DERVIED_TABLE_SQL_FILES_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/download_clickhouse_sql_scripts_py3.py"
+CREATE_DERIVED_TABLES_IN_CLICKHOUSE_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/create_derived_tables_in_clickhouse_database.sh"
+
+# Get the current production database color
+current_production_database_color=$(sh $GET_DB_IN_PROD_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH)
+destination_database_color="unset"
+if [ ${current_production_database_color:0:4} == "blue" ] ; then
+    destination_database_color="green"
+fi
+if [ ${current_production_database_color:0:5} == "green" ] ; then
+    destination_database_color="blue"
+fi
+if [ "$destination_database_color" == "unset" ] ; then
+    echo "Error during determination of the destination database color" >&2
+    exit 1
+fi
+
+# Drop tables from non-production ClickHouse DB to make room for incoming copy
+echo "dropping tables from clickhouse database $destination_database_color..."
+if ! $DROP_TABLES_FROM_CLICKHOUSE_DATABASE_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH $destination_database_color ; then
+    echo "Error during dropping of tables from clickhouse database $destination_database_color" >&2
+    exit 1
+fi
+
+# Use Sling to copy data from non-production MySQL DB to non-production ClickHouse DB
+echo "copying tables from mysql database $destination_database_color to clickhouse database $destination_database_color..."
+if ! $COPY_TABLES_FROM_MYSQL_TO_CLICKHOUSE_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH $destination_database_color ; then
+    echo "Error during copying of tables from mysql database $destination_database_color to clickhouse database $destination_database_color" >&2
+    exit 1
+fi
+
+# Check if derived table sql script dirpath exists
+# If not, try to create it
+derived_table_sql_script_dirpath="$tmp/create_derived_clickhouse_tables"
+if ! [ -e "$derived_table_sql_script_dirpath" ] ; then
+    if ! mkdir -p "$derived_table_sql_script_dirpath" ; then
+        echo "Error : could not create target directory '$derived_table_sql_script_dirpath'" >&2
+        exit 1
+    fi
+fi
+
+# Remove any scripts currently in the derived table sql script dirpath
+if [[ -d "$derived_table_sql_script_dirpath" && "$derived_table_sql_script_dirpath" != "/" ]]; then
+    rm -rf "$derived_table_sql_script_dirpath"/*
+fi
+
+# Attempt to download the derived table SQL files from github
+if ! $DOWNLOAD_DERVIED_TABLE_SQL_FILES_SCRIPT_FILEPATH "$derived_table_sql_script_dirpath" ; then
+    echo "Error : could not download needed derived table construction .sql files from github" >&2
+    exit 1
+fi
+
+# Create the additional derived tables inside of non-production Clickhouse DB
+echo "creating derived tables in clickhouse database $destination_database_color..."
+if ! $CREATE_DERIVED_TABLES_IN_CLICKHOUSE_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH $destination_database_color "$derived_table_sql_script_dirpath"/* ; then
+    echo "Error during derivation of clickhouse tables in clickhouse database $destination_database_color" >&2
+    exit 1
+fi

import-scripts/genie-airflow-import-sql.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Script for running GENIE import
+# Consists of the following:
+# - Import of cancer types
+# - Import from genie-portal column in spreadsheet
+
+IMPORTER=$1
+PORTAL_SCRIPTS_DIRECTORY=$2
+if [ -z $PORTAL_SCRIPTS_DIRECTORY ]; then
+    PORTAL_SCRIPTS_DIRECTORY="/data/portal-cron/scripts"
+fi
+AUTOMATION_ENV_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/automation-environment.sh"
+if [ ! -f $AUTOMATION_ENV_SCRIPT_FILEPATH ] ; then
+    echo "`date`: Unable to locate $AUTOMATION_ENV_SCRIPT_FILEPATH, exiting..."
+    exit 1
+fi
+source $AUTOMATION_ENV_SCRIPT_FILEPATH
+
+# Get the current production database color
+MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/pipelines-credentials/manage_genie_database_update_tools.properties"
+GET_DB_IN_PROD_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/get_database_currently_in_production.sh"
+current_production_database_color=$(sh $GET_DB_IN_PROD_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH)
+destination_database_color="unset"
+if [ ${current_production_database_color:0:4} == "blue" ] ; then
+    destination_database_color="green"
+fi
+if [ ${current_production_database_color:0:5} == "green" ] ; then
+    destination_database_color="blue"
+fi
+if [ "$destination_database_color" == "unset" ] ; then
+    echo "Error during determination of the destination database color" >&2
+    exit 1
+fi
+
+tmp=$PORTAL_HOME/tmp/import-cron-genie
+IMPORTER_JAR_FILENAME="/data/portal-cron/lib/$IMPORTER-aws-importer-$destination_database_color.jar"
+JAVA_IMPORTER_ARGS="$JAVA_SSL_ARGS -Dspring.profiles.active=dbcp -Djava.io.tmpdir=$tmp -ea -cp $IMPORTER_JAR_FILENAME org.mskcc.cbio.importer.Admin"
+ONCOTREE_VERSION="oncotree_2019_12_01"
+
+# Direct importer logs to stdout
+tail -f $PORTAL_HOME/logs/genie-aws-importer.log &
+
+echo "Importing with $IMPORTER_JAR_FILENAME"
+echo "Importing cancer type updates into genie portal database..."
+$JAVA_BINARY -Xmx16g $JAVA_IMPORTER_ARGS --import-types-of-cancer --oncotree-version $ONCOTREE_VERSION
+if [ $? -gt 0 ]; then
+    echo "Cancer type import failed!" >&2
+    exit 1
+fi
+
+echo "Importing study data into genie portal database..."
+$JAVA_BINARY -Xmx64g $JAVA_IMPORTER_ARGS --update-study-data --portal genie-portal --update-worksheet --oncotree-version $ONCOTREE_VERSION --transcript-overrides-source mskcc --disable-redcap-export
+if [ $? -gt 0 ]; then
+    echo "Genie import failed!" >&2
+    exit 1
+fi
+
+num_studies_updated=''
+num_studies_updated_filename="$tmp/num_studies_updated.txt"
+if [ -r "$num_studies_updated_filename" ] ; then
+    num_studies_updated=$(cat "$num_studies_updated_filename")
+fi
+if [[ -z $num_studies_updated ]] || [[ $num_studies_updated == "0" ]] ; then
+    echo "No studies updated, either due to error or failure to mark a study in the spreadsheet" >&2
+    exit 1
+fi
+echo "$num_studies_updated number of studies were updated"

import-scripts/genie-airflow-set-import-status.sh

@@ -0,0 +1,20 @@
+IMPORT_STATUS=$1
+PORTAL_SCRIPTS_DIRECTORY=$2
+if [ -z $PORTAL_SCRIPTS_DIRECTORY ]; then
+    PORTAL_SCRIPTS_DIRECTORY="/data/portal-cron/scripts"
+fi
+AUTOMATION_ENV_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/automation-environment.sh"
+if [ ! -f $AUTOMATION_ENV_SCRIPT_FILEPATH ] ; then
+    echo "`date`: Unable to locate $AUTOMATION_ENV_SCRIPT_FILEPATH, exiting..."
+    exit 1
+fi
+source $AUTOMATION_ENV_SCRIPT_FILEPATH
+
+SET_UPDATE_PROCESS_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/set_update_process_state.sh"
+MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/pipelines-credentials/manage_genie_database_update_tools.properties"
+
+# Update import status
+if ! $SET_UPDATE_PROCESS_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH $IMPORT_STATUS ; then
+    echo "Error during execution of $SET_UPDATE_PROCESS_SCRIPT_FILEPATH : could not set $IMPORT_STATUS state" >&2
+    exit 1
+fi

import-scripts/genie-airflow-setup.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Script for running pre-import steps
+# Consists of the following:
+# - Database check (given a specific importer)
+# - Data fetch from provided sources
+# - Refreshing CDD/Oncotree caches
+
+IMPORTER=$1
+PORTAL_SCRIPTS_DIRECTORY=$2
+if [ -z $PORTAL_SCRIPTS_DIRECTORY ]; then
+    PORTAL_SCRIPTS_DIRECTORY="/data/portal-cron/scripts"
+fi
+AUTOMATION_ENV_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/automation-environment.sh"
+if [ ! -f $AUTOMATION_ENV_SCRIPT_FILEPATH ] ; then
+    echo "`date`: Unable to locate $AUTOMATION_ENV_SCRIPT_FILEPATH, exiting..."
+    exit 1
+fi
+source $AUTOMATION_ENV_SCRIPT_FILEPATH
+
+# Get the current production database color
+MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/pipelines-credentials/manage_genie_database_update_tools.properties"
+GET_DB_IN_PROD_SCRIPT_FILEPATH="$PORTAL_SCRIPTS_DIRECTORY/get_database_currently_in_production.sh"
+current_production_database_color=$(sh $GET_DB_IN_PROD_SCRIPT_FILEPATH $MANAGE_DATABASE_TOOL_PROPERTIES_FILEPATH)
+destination_database_color="unset"
+if [ ${current_production_database_color:0:4} == "blue" ] ; then
+    destination_database_color="green"
+fi
+if [ ${current_production_database_color:0:5} == "green" ] ; then
+    destination_database_color="blue"
+fi
+if [ "$destination_database_color" == "unset" ] ; then
+    echo "Error during determination of the destination database color" >&2
+    exit 1
+fi
+
+tmp=$PORTAL_HOME/tmp/import-cron-genie
+IMPORTER_JAR_FILENAME="/data/portal-cron/lib/$IMPORTER-aws-importer-$destination_database_color.jar"
+echo "Checking using $IMPORTER_JAR_FILENAME"
+JAVA_IMPORTER_ARGS="$JAVA_SSL_ARGS -Dspring.profiles.active=dbcp -Djava.io.tmpdir=$tmp -ea -cp $IMPORTER_JAR_FILENAME org.mskcc.cbio.importer.Admin"
+
+# Database check
+echo "Checking if database version is compatible"
+$JAVA_BINARY $JAVA_IMPORTER_ARGS --check-db-version
+if [ $? -gt 0 ]; then
+    echo "Database version expected by portal does not match version in database!" >&2
+    exit 1
+fi
+
+# Fetch updates in genie repository
+echo "Fetching updates from genie..."
+$JAVA_BINARY $JAVA_IMPORTER_ARGS --fetch-data --data-source genie --run-date latest
+if [ $? -gt 0 ]; then
+    echo "GENIE fetch failed!" >&2
+    exit 1
+fi
+
+# Refresh CDD/Oncotree cache to pull latest metadata
+echo "Refreshing CDD/ONCOTREE caches..."
+bash $PORTAL_SCRIPTS_DIRECTORY/refresh-cdd-oncotree-cache.sh
+if [ $? -gt 0 ]; then
+    echo "Failed to refresh CDD and/or ONCOTREE cache during GENIE import!" >&2
+    exit 1
+fi