Merge pull request #61 from malariagen/align_import_workflows

Align import workflows with changes in #51
malariagen · Nov 25, 2020 · e5ee427 · e5ee427
2 parents 7ca7ca5 + f66a4c1
commit e5ee427
Show file tree

Hide file tree

Showing 25 changed files with 92 additions and 70 deletions.
diff --git a/pipelines/SNP-genotyping-vector/farm5/input_files/small/AV0148-C.json b/pipelines/SNP-genotyping-vector/farm5/input_files/small/AV0148-C.json
@@ -16,7 +16,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "SNPGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...read-alignment-and-genotyping-vector/farm5/BatchImportShortReadAlignmentAndGenotyping.wdl b/...read-alignment-and-genotyping-vector/farm5/BatchImportShortReadAlignmentAndGenotyping.wdl
@@ -81,4 +81,8 @@ workflow BatchImportShortReadAlignmentAndGenotyping {
     Array[File] output_vcf_index = SNPGenotyping.output_vcf_index
     Array[File] zarr_output = SNPGenotyping.zarr_output
   }
+
+  meta {
+    allowNestedInputs: true
+  }
 }
diff --git a/...mport-short-read-alignment-and-genotyping-vector/farm5/input_files/small/batch_size4.json b/...mport-short-read-alignment-and-genotyping-vector/farm5/input_files/small/batch_size4.json
@@ -14,7 +14,7 @@
   "BatchImportShortReadAlignmentAndGenotyping.alleles_vcf": "/nfs/pathogen/malariaGEN/vectors/allele_sites/ag.allsites.nonN.vcf.gz",
   "BatchImportShortReadAlignmentAndGenotyping.alleles_vcf_index": "/nfs/pathogen/malariaGEN/vectors/allele_sites/ag.allsites.nonN.vcf.gz.tbi",
   "BatchImportShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...alignment-and-genotyping-vector/farm5/input_files/validation/batch_size10_validation.json b/...alignment-and-genotyping-vector/farm5/input_files/validation/batch_size10_validation.json
@@ -14,7 +14,7 @@
   "BatchImportShortReadAlignmentAndGenotyping.alleles_vcf": "/nfs/pathogen/malariaGEN/vectors/allele_sites/ag.allsites.nonN.vcf.gz",
   "BatchImportShortReadAlignmentAndGenotyping.alleles_vcf_index": "/nfs/pathogen/malariaGEN/vectors/allele_sites/ag.allsites.nonN.vcf.gz.tbi",
   "BatchImportShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...ent-and-genotyping-vector/farm5/input_files/validation/batch_size28_ag3.2_validation.json b/...ent-and-genotyping-vector/farm5/input_files/validation/batch_size28_ag3.2_validation.json
@@ -14,7 +14,7 @@
   "BatchImportShortReadAlignmentAndGenotyping.alleles_vcf": "/nfs/pathogen/malariaGEN/vectors/allele_sites/ag.allsites.nonN.vcf.gz",
   "BatchImportShortReadAlignmentAndGenotyping.alleles_vcf_index": "/nfs/pathogen/malariaGEN/vectors/allele_sites/ag.allsites.nonN.vcf.gz.tbi",
   "BatchImportShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/pipelines/import-short-read-alignment-vector/farm5/ImportShortReadAlignment.wdl b/pipelines/import-short-read-alignment-vector/farm5/ImportShortReadAlignment.wdl
@@ -130,4 +130,8 @@ workflow ImportShortReadAlignment {
     File samtools_flagstat_report_file = SamtoolsFlagStat.report_file
     File callable_loci_summary_file = GatkCallableLoci.summary_file
   }
+
+  meta {
+    allowNestedInputs: true
+  }
 }
diff --git a/...ines/import-short-read-lanelet-alignment-vector/farm5/ImportShortReadLaneletAlignment.wdl b/...ines/import-short-read-lanelet-alignment-vector/farm5/ImportShortReadLaneletAlignment.wdl
@@ -57,4 +57,8 @@ workflow ImportShortReadLaneletAlignment {
   output {
     Array[File] output_bam = LaneletAlignment.output_bam
   }
+
+  meta {
+    allowNestedInputs: true
+  }
 }
diff --git a/pipelines/short-read-alignment-and-genotyping-vector/farm5/input_files/small/AV0148-C.json b/pipelines/short-read-alignment-and-genotyping-vector/farm5/input_files/small/AV0148-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AA0052-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AA0052-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AB0252-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AB0252-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AC0010-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AC0010-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AJ0037-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AJ0037-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AN0131-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AN0131-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...es/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AN0280-Cx.json b/...es/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AN0280-Cx.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AN0326-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AN0326-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AR0078-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AR0078-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AZ0156-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/AZ0156-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/BL0358-C.json b/...nes/short-read-alignment-and-genotyping-vector/farm5/input_files/validation/BL0358-C.json
@@ -14,7 +14,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignmentAndGenotyping.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/pipelines/short-read-alignment-vector/farm5/input_files/AB0252-C.json b/pipelines/short-read-alignment-vector/farm5/input_files/AB0252-C.json
@@ -13,7 +13,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignment.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/pipelines/short-read-alignment-vector/farm5/input_files/AN0131-C.json b/pipelines/short-read-alignment-vector/farm5/input_files/AN0131-C.json
@@ -13,7 +13,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignment.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/pipelines/short-read-alignment-vector/farm5/input_files/small/AV0079-C.json b/pipelines/short-read-alignment-vector/farm5/input_files/small/AV0079-C.json
@@ -13,7 +13,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignment.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/pipelines/short-read-alignment-vector/farm5/input_files/small/AV0148-C.json b/pipelines/short-read-alignment-vector/farm5/input_files/small/AV0148-C.json
@@ -13,7 +13,7 @@
     "ref_pac": "/nfs/pathogen/malariaGEN/vectors/references/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4/Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa.pac"
   },
   "ShortReadAlignment.runTimeSettings": {
-    "lsf_group": "malaria-dk",
+    "lsf_group": "pathdev",
     "lsf_queue": "normal",
     "bwa_singularity_image": "bwa.0.7.15_v0.1.2.sif",
     "samtools_singularity_image": "samtools.1.4.1_v0.1.2.sif",

diff --git a/tasks/farm5/ImportTasks.wdl b/tasks/farm5/ImportTasks.wdl
@@ -7,6 +7,11 @@ task ImportIRODS {
     String irods_path
     String sample_id
 
+    String singularity_image = runTimeSettings.irods_singularity_image
+    Int num_cpu = 1
+    Int memory = 1000
+    String? lsf_group
+    String? lsf_queue
     RunTimeSettings runTimeSettings
   }
 
@@ -25,11 +30,11 @@ task ImportIRODS {
   }
 
   runtime {
-    singularity: runTimeSettings.irods_singularity_image
-    memory: 1000
-    cpu: "1"
-    lsf_group: select_first([runTimeSettings.lsf_group, "malaria-dk"])
-    lsf_queue: select_first([runTimeSettings.lsf_queue, "normal"])
+    singularity: singularity_image
+    memory: memory
+    cpu: num_cpu
+    lsf_group: select_first([runTimeSettings.lsf_group, lsf_group, "pathdev"])
+    lsf_queue: select_first([runTimeSettings.lsf_queue, lsf_queue, "normal"])
   }
 
   output {
@@ -50,6 +55,11 @@ task BatchSplitUpInputFile {
   input {
     File batch_sample_manifest_file
 
+    String singularity_image = runTimeSettings.binder_singularity_image
+    Int num_cpu = 1
+    Int memory = 3000
+    String? lsf_group
+    String? lsf_queue
     RunTimeSettings runTimeSettings
   }
 
@@ -74,11 +84,11 @@ task BatchSplitUpInputFile {
   >>>
 
   runtime {
-    singularity: runTimeSettings.binder_singularity_image
-    memory: 3000
-    cpu: "1"
-    lsf_group: select_first([runTimeSettings.lsf_group, "malaria-dk"])
-    lsf_queue: select_first([runTimeSettings.lsf_queue, "normal"])
+    singularity: singularity_image 
+    memory: memory
+    cpu: num_cpu
+    lsf_group: select_first([runTimeSettings.lsf_group, lsf_group, "pathdev"])
+    lsf_queue: select_first([runTimeSettings.lsf_queue, lsf_queue, "normal"])
   }
 
   output {

diff --git a/tasks/farm5/SNPGenotypingTasks.wdl b/tasks/farm5/SNPGenotypingTasks.wdl
@@ -59,8 +59,8 @@ task UnifiedGenotyper {
     singularity: singularity_image
     cpu: num_cpu
     memory: memory
-    lsf_group: select_first([lsf_group, runTimeSettings.lsf_group, "malaria-dk"])
-    lsf_queue: select_first([lsf_queue, runTimeSettings.lsf_queue, "normal"])
+    lsf_group: select_first([runTimeSettings.lsf_group, lsf_group, "pathdev"])
+    lsf_queue: select_first([runTimeSettings.lsf_queue, lsf_queue, "normal"])
   }
   output {
     File output_vcf = output_vcf_filename
@@ -106,8 +106,8 @@ task VcfToZarr {
     singularity: singularity_image
     cpu: num_cpu
     memory: memory
-    lsf_group: select_first([lsf_group, runTimeSettings.lsf_group, "malaria-dk"])
-    lsf_queue: select_first([lsf_queue, runTimeSettings.lsf_queue, "normal"])
+    lsf_group: select_first([runTimeSettings.lsf_group, lsf_group, "pathdev"])
+    lsf_queue: select_first([runTimeSettings.lsf_queue, lsf_queue, "normal"])
   }
   output {
     File output_log_file = output_log_file_name