main.yml

name: Build-database

on:
  push:
    branches:
      - "*"
    paths:
      - data/**
      
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: write
  pages: write
  id-token: write

jobs:
# Placeholder
   build-samples:
     #if: github.actor != 'github-actions[bot]'
     runs-on: ubuntu-20.04
     environment: build
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
       - name: Download artifacts from previous drug gen
         uses: actions/download-artifact@v3
       - name: Pull sample image
         run: docker pull sgosline/srp-samplechem
       - name: Run sample-chem python command
         run: |
           docker run -v $PWD:/tmp sgosline/samplechem python build_script.py --samps
       - name: List files cwd
         run: ls -lah 
#       - name: Copy drugs file to artifact 
#         run: |
#           mkdir additive-drugs-file-2
#           mv drugs.tsv ignore_chems.txt additive-drugs-file-2
#       - name: Upload artifacts for genes
#         uses: actions/upload-artifact@v3
#         with:
#           name: additive-drugs-file-2
#           path: additive-drugs-file-2

#   build-drugs-depmap-NCI60-pt1:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-drugs-depmap-CTRPv2-GDSC-gCSI-PRISM-CCLE-FIMM]
#     steps:
#       - name: Checkout Repo
#         uses: actions/checkout@v3
#       - name: Download artifacts from previous drug gen
#         uses: actions/download-artifact@v3
#         with:
#           name: additive-drugs-file-2
#       - name: List files cwd
#         run: ls -lah 
#       # - name: Install R
#       #   uses: r-lib/actions/setup-r@v2
#       # - name: Install R and Python Packages
#       #   run: |
#       #     Rscript build/depmap/requirements.r
#       #     pip install -r build/depmap/requirements.txt
#       - name: Pull depmap image
#         run: docker pull jjacobson95/depmap
#       - name: Run depmap NCI60 Drug retrieval
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#       # - name: Run depmap NCI60 Drug retrieval
#       #   run:  Rscript build/depmap/CreateDrugFileCL.R NCI60
#       - name: Copy drugs file to artifact 
#         run: |
#           mkdir additive-drugs-file-3
#           mv drugs.tsv ignore_chems.txt additive-drugs-file-3
#       - name: Upload artifacts for genes
#         uses: actions/upload-artifact@v2
#         with:
#           name: additive-drugs-file-3
#           path: additive-drugs-file-3


#   build-drugs-depmap-NCI60-pt2:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-drugs-depmap-NCI60-pt1]
#     steps:
#       - name: Checkout Repo
#         uses: actions/checkout@v2
#       - name: Download artifacts from previous drug gen
#         uses: actions/download-artifact@v2
#         with:
#           name: additive-drugs-file-3
#       - name: List files cwd
#         run: ls -lah 
#       - name: Pull depmap image
#         run: docker pull jjacobson95/depmap
#       - name: Run depmap NCI60 Drug retrieval
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#       - name: Copy drugs file to artifact 
#         run: |
#           mkdir additive-drugs-file-4
#           mv drugs.tsv ignore_chems.txt additive-drugs-file-4
#       - name: Upload artifacts for genes
#         uses: actions/upload-artifact@v2
#         with:
#           name: additive-drugs-file-4
#           path: additive-drugs-file-4


#   build-drugs-depmap-NCI60-pt3:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-drugs-depmap-NCI60-pt2]
#     steps:
#       - name: Checkout Repo
#         uses: actions/checkout@v2
#       - name: Download artifacts from previous drug gen
#         uses: actions/download-artifact@v2
#         with:
#           name: additive-drugs-file-4
#       - name: List files cwd
#         run: ls -lah 
#       - name: Pull depmap image
#         run: docker pull jjacobson95/depmap
#       - name: Run depmap NCI60 Drug retrieval
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#       - name: Copy drugs file to artifact 
#         run: |
#           mkdir additive-drugs-file-5
#           mv drugs.tsv ignore_chems.txt additive-drugs-file-5
#       - name: Upload artifacts for genes
#         uses: actions/upload-artifact@v2
#         with:
#           name: additive-drugs-file-5
#           path: additive-drugs-file-5


#   build-drugs-depmap-NCI60-pt4:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-drugs-depmap-NCI60-pt3]
#     steps:
#       - name: Checkout Repo
#         uses: actions/checkout@v2
#       - name: Download artifacts from previous drug gen
#         uses: actions/download-artifact@v2
#         with:
#           name: additive-drugs-file-5
#       - name: List files cwd
#         run: ls -lah 
#       - name: Pull depmap image
#         run: docker pull jjacobson95/depmap
#       - name: Run depmap NCI60 Drug retrieval
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#       - name: Copy drugs file to artifact 
#         run: |
#           mkdir additive-drugs-file-6
#           mv drugs.tsv ignore_chems.txt additive-drugs-file-6
#       - name: Upload artifacts for genes
#         uses: actions/upload-artifact@v2
#         with:
#           name: additive-drugs-file-6
#           path: additive-drugs-file-6

#   # build-drugs-depmap-NCI60-pt5:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt4]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-6
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-7
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-7
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-7
#   #         path: additive-drugs-file-7

#   # build-drugs-depmap-NCI60-pt6:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt5]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-7
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-8
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-8
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-8
#   #         path: additive-drugs-file-8

#   # build-drugs-depmap-NCI60-pt7:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt6]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-8
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-9
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-9
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-9
#   #         path: additive-drugs-file-9

#   # build-drugs-depmap-NCI60-pt8:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt7]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-9
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-10
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-10
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-10
#   #         path: additive-drugs-file-10

#   # build-drugs-depmap-NCI60-pt9:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt8]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-10
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-11
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-11
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-11
#   #         path: additive-drugs-file-11

#   # build-drugs-depmap-NCI60-pt10:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt9]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-11
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-12
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-12
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-12
#   #         path: additive-drugs-file-12

#   # build-drugs-depmap-NCI60-pt11:
#   #   if: github.actor != 'github-actions[bot]'
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   needs: [build-drugs-depmap-NCI60-pt10]
#   #   steps:
#   #     - name: Checkout Repo
#   #       uses: actions/checkout@v2
#   #     - name: Download artifacts from previous drug gen
#   #       uses: actions/download-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-12
#   #     - name: List files cwd
#   #       run: ls -lah 
#   #     - name: Pull depmap image
#   #       run: docker pull jjacobson95/depmap
#   #     - name: Run depmap NCI60 Drug retrieval
#   #       run: |
#   #         docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60
#   #     - name: Copy drugs file to artifact 
#   #       run: |
#   #         mkdir additive-drugs-file-13
#   #         mv drugs.tsv ignore_chems.txt additive-drugs-file-13
#   #     - name: Upload artifacts for genes
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: additive-drugs-file-13
#   #         path: additive-drugs-file-13

#   build-drugs-depmap-capture-stragglers:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-drugs-depmap-NCI60-pt4]
#     steps:
#       - name: Checkout Repo
#         uses: actions/checkout@v2
#       - name: Download artifacts from previous drug gen
#         uses: actions/download-artifact@v2
#         with:
#           name: additive-drugs-file-6
#       - name: List files cwd
#         run: ls -lah 
#       - name: List files cwd
#         run: rm ignore_chems.txt
#       - name: Pull depmap image
#         run: docker pull jjacobson95/depmap
#       - name: Run depmap straggler Drug retrieval
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R CTRPv2,GDSC,gCSI,PRISM,CCLE,FIMM,NCI60
#       - name: Copy drugs file to artifact 
#         run: |
#           mkdir additive-drugs-file-7
#           mv drugs.tsv ignore_chems.txt additive-drugs-file-7
#       - name: Upload artifacts for genes
#         uses: actions/upload-artifact@v2
#         with:
#           name: additive-drugs-file-7
#           path: additive-drugs-file-7

#   build-cptac-samples:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-depmap-samples,cptac-docker-build]
#     steps:
#       #Add upload artifact from CPTAC Samples 
#       - name: Checkout  CPTAC
#         uses: actions/checkout@v2
#       - name: Download artifacts from depmap
#         uses: actions/download-artifact@v2
#         with:
#           name: depmap-samples
#       - name: Download artifacts from DepMap (genes)
#         uses: actions/download-artifact@v2  
#         with:
#           name: genes-file
#       # - name: Copy DepMap sample from build location. 
#       #   run: |
#       #     cp build/cptac/requirements.txt .
#       #     cp build/cptac/*.py .
#       #     # copy build/genes.csv .
#       - name: Pull cptac image
#         run: docker pull jjacobson95/cptac
#       - name: Run CPTAC samples Container
#         run: docker run -v $PWD:/tmp jjacobson95/cptac --geneFile=/tmp/genes.csv --prevSampleFile=/tmp/depmap_samples.csv 
#       - name: Copy file from CPTAC Samples Container
#         run: |
#           mkdir cptac-samples
#           mv cptac_samples.csv cptac-samples
#       - name: Upload artifacts for CPTAC
#         uses: actions/upload-artifact@v2
#         with:
#           name: cptac-samples
#           path: cptac-samples

#   build-hcmi-samples:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-cptac-samples,hcmi-docker-build]
#     steps:
#       - name: Checkout  HCMI
#         uses: actions/checkout@v2
#       - name: Download artifacts from CPTAC
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-samples
#       # - name: Build HCMI Image
#       #   run: docker build -t hcmi-samples-builder -f build/docker/Dockerfile.hcmi .
#       - name: Pull hcmi image
#         run: docker pull jjacobson95/hcmi
#       - name: Run HCMI samples Container
#         run: docker run -v $PWD:/tmp --name hcmi-samples-container jjacobson95/hcmi python 01-createHCMISamplesFile.py
#       - name: Copy file from HCMI Samples Container
#         run: |
#           mkdir hcmi-samples
#           for file in hcmi_samples.csv; do
#             docker cp hcmi-samples-container:/usr/src/app/$file hcmi-samples/$file
#           done
#       - name: Get HCMI container logs
#         run: docker logs hcmi-samples-container 
#       - name: Upload artifacts for HCMI
#         uses: actions/upload-artifact@v2
#         with:
#           name: hcmi-samples
#           path: hcmi-samples

#   build-beataml-samples:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-hcmi-samples,beataml-docker-build]
#     steps:
#       #Add upload artifact from CPTAC Samples 
#       - name: Checkout  BeatAML
#         uses: actions/checkout@v2
#       - name: Download artifacts from HCMI
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-samples
#       - name: Pull beataml image
#         run: docker pull jjacobson95/beataml
#       - name: Run BeatAML Container with samples option
#         run: docker run -v $PWD:/tmp --name beataml-samples-container jjacobson95/beataml python GetBeatAML.py --token ${{ secrets.SYNAPSE_TOKEN_SECRET }} --samples
#       - name: Copy file from BeatAML Samples Container
#         run: |
#           mkdir beataml-samples
#           for file in beataml_samples.csv; do
#             docker cp beataml-samples-container:/usr/src/app/$file beataml-samples/$file
#           done
#       - name: Get BeatAML container logs
#         run: docker logs beataml-samples-container 
#       - name: Upload artifacts for BeatAML
#         uses: actions/upload-artifact@v2
#         with:
#           name: beataml-samples
#           path: beataml-samples

# # Placeholder
#   build-mpnst-samples:
#     if: github.actor != 'github-actions[bot]'
#     runs-on: ubuntu-20.04
#     environment: build
#     needs: [build-beataml-samples,mpnst-docker-build]
#     steps:
#       #Add upload artifact from CPTAC Samples 
#       - name: Checkout  MPNST
#         uses: actions/checkout@v2

#   build-hcmi:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-hcmi-samples]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout  HCMI
#         uses: actions/checkout@v2
#       - name: Download artifacts from HCMI
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-samples
#       - name: List files cwd
#         run: ls -lah 
#       # - name: Build HCMI Image
#       #   run: docker build -t hcmi-builder -f build/docker/Dockerfile.hcmi .
#       - name: Pull hcmi image
#         run: docker pull jjacobson95/hcmi
#       - name: Run HCMI Container
#         run: |
#           docker run -v $PWD:/tmp --name hcmi-container jjacobson95/hcmi /bin/bash -c "python 02-getHCMIData.py -m full_manifest.txt -t transcriptomics -o hcmi_transcriptomics.csv && python 02-getHCMIData.py -m full_manifest.txt -M full_manifest_files -t copy_number -o hcmi_copy_number.csv && python 02-getHCMIData.py -m full_manifest.txt -M full_manifest_files -t mutations -o hcmi_mutations.csv"
#       - name: Copy files from HCMI Container
#         run: |
#           mkdir hcmi-files
#           for file in hcmi_transcriptomics.csv hcmi_copy_number.csv hcmi_mutations.csv; do
#             docker cp hcmi-container:/usr/src/app/$file hcmi-files/$file
#           done
#           cp hcmi_samples.csv hcmi-files
#       - name: Get HCMI container logs
#         run: docker logs hcmi-container
#       - name: Upload artifacts for HCMI
#         uses: actions/upload-artifact@v2
#         with:
#           name: hcmi-files
#           path: hcmi-files

#   check-hcmi-schema:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-hcmi]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code HCMI
#         uses: actions/checkout@v2
#       - name: Download artifacts from HCMI Samples
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-samples
#       - name: Download artifacts from HCMI
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-files
#       - name: Download linkml
#         run: |
#             pip install linkml
#             pip install linkml-validator
#       - name: check dir
#         run: ls
#       - name: Run linkml schema checker
#         run: bash schema/check_hcmi_linkml.sh

#   build-beataml:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-beataml-samples,build-drugs-depmap-capture-stragglers]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code BeatAML
#         uses: actions/checkout@v2
#       - name: Download artifacts from Beataml Samples
#         uses: actions/download-artifact@v2
#         with:
#           name: beataml-samples
#       - name: Download artifacts from Drugs
#         uses: actions/download-artifact@v2  
#         with:
#           name: additive-drugs-file-7
#       - name: List files cwd
#         run: ls -lah 
#       - name: Pull beataml image
#         run: docker pull jjacobson95/beataml
#       - name: Run BeatAML Container
#         run: docker run -v $PWD:/tmp --name beataml-container jjacobson95/beataml python GetBeatAML.py --token ${{ secrets.SYNAPSE_TOKEN_SECRET }}
#       - name: Copy files from BeatAML Container
#         run: |
#           mkdir beataml-files
#           for file in beataml_transcriptomics.csv beataml_proteomics.csv beataml_mutations.csv beataml_drugs.tsv beataml_experiments.csv; do
#             docker cp beataml-container:/usr/src/app/$file beataml-files/$file
#           done
#           cp beataml_samples.csv beataml-files
#       - name: Get BeatAML container logs
#         run: docker logs beataml-container
#       - name: Upload artifact for BeatAML
#         uses: actions/upload-artifact@v2
#         with:
#           name: beataml-files
#           path: beataml-files


#   check-beataml-schema:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-beataml]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code BeatAML
#         uses: actions/checkout@v2
#       - name: Download artifacts from Beataml Samples
#         uses: actions/download-artifact@v2
#         with:
#           name: beataml-samples
#       - name: Download artifacts from Beataml 
#         uses: actions/download-artifact@v2
#         with:
#           name: beataml-files
#       - name: Download linkml
#         run: |
#             pip install linkml
#             pip install linkml-validator
#       - name: check dir
#         run: ls
#       - name: Run linkml schema checker
#         run: bash schema/check_beataml_linkml.sh

#   # build-depmap:
#   #   if: github.actor != 'github-actions[bot]'
#   #   needs: [build-depmap-samples]
#   #   runs-on: ubuntu-20.04
#   #   environment: build
#   #   steps:
#   #     - name: Checkout code DepMap
#   #       uses: actions/checkout@v2
#   #     - name: List files cwd before
#   #       run: ls -lah 
#   #     - name: Install coderdata
#   #       run: pip install coderdata
#   #     - name: Download depmap data from coder data
#   #       run: coderdata download --prefix depmap
#   #     - name: List files cwd after
#   #       run: ls -lah 
#   #     - name: Copy files from DepMap Container
#   #       run: |
#   #         mkdir depmap-files
#   #         for file in *depmap*sv*; do
#   #           cp $file depmap-files/$file
#   #         done
#   #     - name: Upload artifact for DepMap
#   #       uses: actions/upload-artifact@v2
#   #       with:
#   #         name: depmap-files
#   #         path: depmap-files

#   build-depmap:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-depmap-samples,build-drugs-depmap-capture-stragglers]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout  DepMap
#         uses: actions/checkout@v2
#       # - name: Copy DepMap sample from build location. 
#       #   run: |
#       #     cp build/depmap/requirements.txt .
#       #     cp build/depmap/requirements.r .
#       #     cp build/depmap/*.R .
#       #     cp build/depmap/*.py .
#       #     cp build/utils/* .
#       - name: Download artifacts from DepMap (samples)
#         uses: actions/download-artifact@v2  
#         with:
#           name: depmap-samples
#       - name: Download artifacts from DepMap (genes)
#         uses: actions/download-artifact@v2  
#         with:
#           name: genes-file
#       - name: Download artifacts from Drugs
#         uses: actions/download-artifact@v2  
#         with:
#           name: additive-drugs-file-7
#       - name: gzip drugs file
#         run: gzip drugs.tsv
#       - name: List files cwd
#         run: ls -lah 
#       - name: cat reqs
#         run: cat requirements.txt
#       - name: Pull depmap image
#         run: docker pull jjacobson95/depmap
#       # - name: docker save space and stats
#       #   run: |
#       #     docker image prune -f 
#       #     docker container prune -f 
#       #     docker volume prune -f 
#       #     docker network prune -f 
#       #     docker system df
#       - name: Run depmap 02-pullDepMap.R Container
#         run: |
#           docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/02-pullDepMap.R /tmp/genes.csv /tmp/depmap_samples.csv
#       - name: Run depmap 02b-pullSanger.R Container
#         run: |
#           docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/02b-pullSanger.R /tmp/genes.csv /tmp/depmap_samples.csv
#       # - name: Run depmap 03-createDrugFile.RContainer
#       #   run: |
#       #     docker run -v $PWD:/tmp depmap Rscript 03-createDrugFile.R CTRPv2,GDSC,gCSI,PRISM,CCLE,FIMM,NCI60
#       - name: Run depmap 04-drug_dosage_and_curves.py Container
#         run: |
#           docker run -v $PWD:/tmp/ jjacobson95/depmap /opt/venv/bin/python /app/04-drug_dosage_and_curves.py --drugfile=/tmp/drugs.tsv.gz --curSampleFile=/tmp/depmap_samples.csv
#       - name: Run depmap 01b-pullDrugs_LINCS.py Container
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap /opt/venv/bin/python /app/01b-pullDrugs_LINCS.py --drugFile /tmp/drugs.tsv.gz
#       - name: Run depmap 04-pullDrugs_LINCS.py Container
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/depmap /opt/venv/bin/python /app/04-pullDrugs_LINCS.py --drugFile /tmp/drugs.tsv.gz
#       - name: Run depmap 05-LINCS_perturbations.py Container
#         run: |
#           docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/05-LINCS_perturbations.R /tmp/genes.csv /tmp/drugs.tsv.gz /tmp/depmap_samples.csv
#       - name: List files cwd
#         run: ls -lah 
#       - name: Copy files from DepMap Container
#         run: |
#           mkdir depmap-files
#           cp depmap_* depmap-files
#       - name: Upload artifacts for DepMap
#         uses: actions/upload-artifact@v2
#         with:
#           name: depmap-files
#           path: depmap-files

#   check-depmap-schema:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-depmap]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code DepMap 
#         uses: actions/checkout@v2
#       - name: Download artifacts from DepMap Samples
#         uses: actions/download-artifact@v2
#         with:
#           name: depmap-samples
#       - name: Download artifacts from DepMap 
#         uses: actions/download-artifact@v2
#         with:
#           name: depmap-files
#       - name: Download linkml
#         run: |
#             pip install linkml
#             pip install linkml-validator
#       - name: check dir
#         run: ls
#       - name: Run linkml schema checker
#         run: bash schema/check_depmap_linkml.sh


#   build-cptac:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-cptac-samples]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout  CPTAC
#         uses: actions/checkout@v2
#       - name: Download artifacts from CPTAC
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-samples
#       - name: Download artifacts from DepMap (genes)
#         uses: actions/download-artifact@v2  
#         with:
#           name: genes-file
#       - name: Copy files from build location. 
#         run: |
#           cp build/cptac/*.py .
#           cp build/cptac/requirements.txt .
#           # copy build/genes.csv .
#       - name: List files cwd
#         run: ls -lah 
#       - name: Pull cptac image
#         run: docker pull jjacobson95/cptac
#       - name: Run CPTAC Container
#         run: |
#           docker run -v $PWD:/tmp jjacobson95/cptac --geneFile=/tmp/genes.csv --curSampleFile=/tmp/cptac_samples.csv   
#       - name: Copy files from CPTAC Container
#         run: |
#           mkdir cptac-files
#           cp cptac_* cptac-files
#       - name: Upload artifacts for CPTAC
#         uses: actions/upload-artifact@v2
#         with:
#           name: cptac-files
#           path: cptac-files

#   check-cptac-schema:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-cptac]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code CPTAC
#         uses: actions/checkout@v2
#       - name: Download artifacts from CPTAC Samples
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-samples
#       - name: Download artifacts from CPTAC
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-files
#       - name: Download linkml and gunzip files
#         run: |
#             pip install linkml
#             pip install linkml-validator
#             find . -type f -name '*.gz' -exec gunzip {} +
#       - name: check dir
#         run: ls
#       - name: Run linkml schema checker
#         run: bash schema/check_cptac_linkml.sh

# #placeholder
#   build-mpnst:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-mpnst-samples]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code MPNST
#         uses: actions/checkout@v2

# # Placeholder
#   check-mpnst-schema:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-mpnst]
#     runs-on: ubuntu-20.04
#     environment: build
#     steps:
#       - name: Checkout code MPNST
#         uses: actions/checkout@v2
#       # - name: Download artifacts from MPNST Samples
#       #   uses: actions/download-artifact@v2
#       #   with:
#       #     name: mpnst-samples
#       # - name: Download artifacts from MPNST
#       #   uses: actions/download-artifact@v2
#       #   with:
#       #     name: mpnst-files
#       # - name: Download linkml
#       #   run: |
#       #       pip install linkml
#       #       pip install linkml-validator
#       # - name: check dir
#       #   run: ls
#       # - name: Run linkml schema checker
#       #   run: bash schema/check_mpnst_linkml.sh

#   data-visualization:
#     if: github.actor != 'github-actions[bot]'
#     needs: [check-hcmi-schema, check-beataml-schema, check-depmap-schema, check-cptac-schema, check-mpnst-schema]
#     environment: build
#     runs-on: ubuntu-20.04
#     steps:
#       - name: Set up R
#         uses: r-lib/actions/setup-r@v2
#       - name: Install R packages
#         run: |
#           R -e "install.packages('RColorBrewer', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('circlize', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('dplyr', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('readr', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('tools', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('ggplot2', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('reshape2', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#           R -e "install.packages('ComplexUpset', dependencies=TRUE, repos='http://cran.rstudio.com/')"
#       - name: checkout repo content
#         uses: actions/checkout@v2
#       - name: Download artifacts from HCMI
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-files
#       - name: Download artifacts from BeatAML
#         uses: actions/download-artifact@v2
#         with:
#           name: beataml-files
#       - name: Download artifacts from DepMap
#         uses: actions/download-artifact@v2
#         with:
#           name: depmap-files
#       # - name: Download artifacts from MPNST
#       #   uses: actions/download-artifact@v2
#       #   with:
#       #     name: mpnst-files
#       - name: Download artifacts from cptac
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-files
#       # - name: Install coderdata and skip build steps
#       #   run: pip install coderdata
#       # - name: Download cptac data from coder data
#       #   run: coderdata download --prefix cptac
#       # - name: Download depmap data from coder data
#       #   run: coderdata download --prefix depmap
#       # - name: Download all data from coder data
#       #   run: coderdata download 
#       - name: Create all-files directory
#         run: mkdir all-files
#       - name: List files cwd
#         run: ls -lah 
#       - name: List files dataSummary dir
#         run: ls -lah dataSummary
#       - name: Gzip all non-samples csv files in current dir.
#         run: |
#           find . -type f -name '*.csv' ! -name '*samples*' -exec gzip -k "{}" +
#       - name: Run Visualization01
#         if: success() || failure()
#         run: Rscript dataSummary/visualization01.R
#       - name: Run Visualization02
#         if: success() || failure()
#         run: Rscript dataSummary/visualization02.R
#       - name: setup data-vis dir
#         if: success() || failure()
#         run: |
#           mkdir -p data-vis
#           rsync -av --remove-source-files --include='*.png' --include='*.pdf' --include='*table.csv' --exclude='*' . data-vis/
#       - name: Upload artifacts
#         if: success() || failure()
#         uses: actions/upload-artifact@v2
#         with:
#           name: data-vis
#           path: data-vis

#   data-visualization-save-to-docs:
#     if: github.actor != 'github-actions[bot]'
#     needs: [data-visualization]
#     environment: build
#     runs-on: ubuntu-20.04
#     steps:
#       - name: Checkout repo content
#         uses: actions/checkout@v2
#       - name: Setup Git config and pull
#         run: |
#           git config user.name "jjacobson95"
#           git config user.email "jeremy.jacobson3402@gmail.com"
#           git pull
#       - name: Download artifacts from data-vis
#         uses: actions/download-artifact@v2
#         with:
#           name: data-vis
#       - name: List files cwd
#         run: ls -lah 
#       - name: List files .. 
#         run: ls -lah .. 
#       - name: List files all 
#         run: ls -lah * 
#       - name: Move files to docs directory
#         run: |
#           rsync -av --remove-source-files --include='*.png' --include='*.pdf' --exclude='*' . docs/assets/stats
#           rsync -av --remove-source-files --include='*table.csv' --exclude='*' . docs/_data
#       #    rsync -av --remove-source-files --include='*.png' --include='*.pdf' --include='*table.csv' --exclude='*' . docs/
#       - name: List files in docs after moving
#         run: ls -lah docs
#       - name: List files in local after moving
#         run: ls -lah 
#       - name: Git add and push new changes. 
#         run: |
#           git add -f docs/*
#           git status
#           if git diff --staged --quiet; then
#             echo "No changes to commit"
#           else
#             git commit -m "Add data visualization artifacts to docs"
#             git push
#           fi
#   gen-stats-and-push:
#     if: github.actor != 'github-actions[bot]'
#     needs: [build-hcmi, build-beataml, build-depmap, build-cptac, build-mpnst]
#     environment: build
#     runs-on: ubuntu-20.04
#     steps:
#       - name: Checkout repo content
#         uses: actions/checkout@v2
#       - name: Setup Git config and pull
#         run: |
#           git config user.name "jjacobson95"
#           git config user.email "jeremy.jacobson3402@gmail.com"
#           git pull
#       - name: Download artifacts from HCMI
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-files
#       - name: Download artifacts from BeatAML
#         uses: actions/download-artifact@v2
#         with:
#           name: beataml-files
#       - name: Download artifacts from DepMap
#         uses: actions/download-artifact@v2
#         with:
#           name: depmap-files
#       # If this pipeline is supported in the future, Gen stats has been updated to include mpnst too. 
#       # - name: Download artifacts from MPNST
#       #   uses: actions/download-artifact@v2
#       #   with:
#       #     name: mpnst-files
#       - name: Download artifacts from cptac
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-files
#       - name: Install coderdata
#         run: |
#           pip install coderdata
#           pip install pyyaml
#       - name: Run stats generation script
#         run: |
#           python scripts/gen_stats.py
#       - name: Git add and push new changes if they exist. 
#         run: |
#           git pull
#           mv stats.yml docs/_data/stats.yml
#           git add -f docs/_data/stats.yml
#           git status
#           if git diff --staged --quiet; then
#             echo "No changes to commit"
#           else
#             git commit -m "Add stats.yml to docs/_data"
#             git push
#           fi

#   upload-to-figshare:
#     if: github.actor != 'github-actions[bot]'
#     needs: [check-hcmi-schema, check-beataml-schema, check-depmap-schema, check-cptac-schema, check-mpnst-schema]
#     # needs: [build-hcmi, build-beataml, build-depmap, build-cptac, build-mpnst]
#     environment: build
#     runs-on: ubuntu-20.04
#     steps:
#       - name: checkout repo content
#         uses: actions/checkout@v2
#       - name: Download artifacts from HCMI
#         uses: actions/download-artifact@v2
#         with:
#           name: hcmi-files
#       - name: Download artifacts from BeatAML
#         uses: actions/download-artifact@v2
#         with:
#           name: beataml-files
#       - name: Download artifacts from CPTAC
#         uses: actions/download-artifact@v2
#         with:
#           name: cptac-files
#       - name: Download artifacts from DepMap
#         uses: actions/download-artifact@v2
#         with:
#           name: depmap-files
#       # - name: Download artifacts from MPNST
#       #   uses: actions/download-artifact@v2
#       #   with:
#       #     name: mpnst-files
#       - name: Create all-files directory
#         run: mkdir all-files
#       - name: List files cwd
#         run: ls -lah 
#       - name: Move files to all-files
#         run: |
#           rsync -av --remove-source-files --include='*.csv' --include='*.tsv' --include='*.gz' --exclude='*' . all-files/
#       - name: gzip all files in all-files
#         run: gzip all-files/*
#       - name: gunzip all samples files in all-files
#         run: gunzip all-files/*samples*
#       - name: List files in all-files
#         run: ls -lah all-files/
#       # - name: Upload to Figshare
#       #   # This can be updated to trigger with tag releases.
#       #   # run: python scripts/push_to_figshare.py --directory all-files --title "CODERData${{ github.event.release.tag_name }}" --token ${{ secrets.FIGSHARE_TOKEN_SECRET }} -p False
#       #   #This is a short term alternative. Links to branch. Not as pretty but okay for now.
        
#       #   #To do: Extract version number from setup.py and run there. This can occur after merge with package dev branch.
#       #   run: python scripts/push_to_figshare.py --directory all-files --title "CODERData${{ github.ref }}" --token ${{ secrets.FIGSHARE_TOKEN_SECRET }} --project_id 189342 --publish 
#       - name: Extract version from setup.py
#         id: version
#         run: echo "PACKAGE_VERSION=$(grep 'version=' setup.py | sed -n "s/.*version='\(.*\)'.*/\1/p")" >> $GITHUB_ENV
#       - name: Upload to Figshare
#         run: python scripts/push_to_figshare.py --directory all-files --title "CODERData${{ env.PACKAGE_VERSION }}" --token ${{ secrets.FIGSHARE_TOKEN_SECRET }} --project_id 189342 --publish
#       - name: List files in all-files
#         run: ls -lah all-files/
#       - name: Setup Git config
#         run: |
#           git config user.name "jjacobson95"
#           git config user.email "jeremy.jacobson3402@gmail.com"
#       - name: Commit and push changes
#         run: |
#           git pull
#           mv figshare_latest.yml docs/_data
#           git add -f docs/*
#           git status
#           git commit -m "Add figshare_latest.yml to docs/_data"
#           git push


  # update-package-download-function:
  #   if: github.actor != 'github-actions[bot]'
  #   needs: [upload-to-figshare]
  #   environment: build
  #   runs-on: ubuntu-20.04
  #   steps:
  #     - name: checkout repo content
  #       uses: actions/checkout@v2
  #     - name: Update figshare path in downloader.py
  #       run: |
  #         python scripts/update_download_function.py -y docs/_data/figshare_latest.yml -d coderdata/download/downloader.py
  #         cat coderdata/download/downloader.py
  #     - name: Setup Git config
  #       run: |
  #         git config user.name "jjacobson95"
  #         git config user.email "jeremy.jacobson3402@gmail.com"
  #     - name: Commit and push changes
  #       run: |
  #         git pull
  #         git add coderdata/download/downloader.py
  #         git commit -m "update downloader.py"
  #         git push

  # run-tests:
  #   if: github.actor != 'github-actions[bot]'
  #   needs: [update-package-download-function]
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       test-file: [
  #         'test_download_and_load_beataml.py', 
  #         'test_download_and_load_depmap.py', 
  #         'test_download_beataml_cli.py', 
  #         'test_download_depmap_cli.py',
  #         'test_download_and_load_cptac.py', 
  #         'test_download_and_load_hcmi.py', 
  #         'test_download_cptac_cli.py', 
  #         'test_download_hcmi_cli.py',
  #         'test_reformat_datasets.py',
  #         'test_reload_one.py',
  #         'test_reload_all.py',
  #         'test_join_datasets.py',
  #         'test_info.py'
  #       ]
  #   outputs:
  #     result: ${{ steps.test-outcome.outcome }}
  #   steps:
  #   - uses: actions/checkout@v2
  #     with:
  #       fetch-depth: 0
  #   - name: Set up Python
  #     uses: actions/setup-python@v2
  #     with:
  #       python-version: '3.10'
  #   - name: Install dependencies
  #     run: |
  #       python -m pip install --upgrade pip
  #       pip install -r requirements.txt
  #       pip install pytest-xdist
  #   - name: Install as package
  #     run: |
  #       pip install .
  #   - name: Run test
  #     id: test-outcome
  #     run: |
  #       pytest -n 4 tests/${{ matrix.test-file }}

  # build-and-publish:
  #   if: github.actor != 'github-actions[bot]'
  #   needs: [run-tests]
  #   runs-on: ubuntu-latest
  #   environment: build
  #   steps:
  #   - uses: actions/checkout@v2
  #     with:
  #       fetch-depth: 0

  #   - name: Fetch tags from main branch
  #     run: |
  #       git fetch origin 'refs/tags/*:refs/tags/*'

  #   - name: Set up Python
  #     uses: actions/setup-python@v2
  #     with:
  #       python-version: '3.10'

  #   - name: Install dependencies
  #     run: |
  #       python -m pip install --upgrade pip
  #       pip install setuptools wheel twine packaging

  #   - name: Update version
  #     run: |
  #       # Extract the current version from setup.py
  #       CURRENT_VERSION=$(grep -oP "version='\K[0-9]+\.[0-9]+\.[0-9]+" setup.py)
  #       echo "Current version: $CURRENT_VERSION"

  #       # Fetch tags and get the latest tag, removing the 'v' prefix
  #       git fetch --tags
  #       LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1` | sed 's/^v//')
  #       echo "Latest tag: $LATEST_TAG"

  #       # Use Python to compare versions
  #       HIGHER_VERSION=$(python -c "from packaging.version import parse as parse_version; \
  #                                   current = parse_version('$CURRENT_VERSION'); \
  #                                   latest = parse_version('$LATEST_TAG'); \
  #                                   print(current if current > latest else latest)")
  #       echo "Higher version: $HIGHER_VERSION"

  #       # If current version is higher, increment patch version in setup.py
  #       if [ "$CURRENT_VERSION" == "$HIGHER_VERSION" ]; then
  #         IFS='.' read -r -a VERSION_PARTS <<< "$CURRENT_VERSION"
  #         PATCH=$((VERSION_PARTS[2] + 1))
  #         NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$PATCH"
  #         echo "Incremented version: $NEW_VERSION"
  #         sed -i "s/version='\([0-9]\+\.[0-9]\+\.\)[0-9]\+'/version='\1$PATCH'/" setup.py

  #         # Commit and push the updated setup.py
  #         git config --local user.email "action@github.com"
  #         git config --local user.name "GitHub Action"
  #         git pull
  #         git add setup.py
  #         git commit -m "Increment version number to $NEW_VERSION [skip ci]"
  #         git push
  #       else
  #         echo "No version increment needed"
  #       fi
  #   - name: Build local and publish to PyPI
  #     env:
  #       TWINE_USERNAME: __token__
  #       TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN_SECRET }}
  #     run: |
  #       python setup.py sdist bdist_wheel
  #       twine upload dist/* --verbose