Merge pull request nf-core#56 from LouisLeNezet/pipeline_test

Pipeline test
LouisLeNezet · May 27, 2024 · e3dc54d · e3dc54d
2 parents 98c5400 + 06b77b8
commit e3dc54d
Show file tree

Hide file tree

Showing 45 changed files with 1,393 additions and 366 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,26 +11,27 @@ on:
 
 env:
   NXF_ANSI_LOG: false
+  NFTEST_VER: "0.8.4"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
   test:
-    name: Run pipeline with test data
+    name: Run nf-test with standard profiles
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/phaseimpute') }}"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
         TEST_PROFILE:
-          - "test"
-          - "test_sim"
-          - "test_quilt"
+          - "test_all"
+          - "test_validate"
           - "test_stitch"
     steps:
       - name: Check out pipeline code
@@ -44,9 +45,11 @@ jobs:
       - name: Disk space cleanup
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
+      - name: Install nf-test
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
+          sudo mv nf-test /usr/local/bin/
+
       - name: Run pipeline with test data
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile "${{ matrix.TEST_PROFILE }}",docker --outdir ./results
+          nf-test test --tag "${{ matrix.TEST_PROFILE }}" --profile docker --verbose
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@ Initial release of nf-core/phaseimpute, created with the [nf-core](https://nf-co
 - [#19](https://github.com/nf-core/phaseimpute/pull/19) - Changed reference panel to accept a csv, update modules and subworkflows (glimpse1/2 and shapeit5)
 - [#40](https://github.com/nf-core/phaseimpute/pull/40) - Add STITCH method. Reorganize panelprep subworkflows.
 - [#51](https://github.com/nf-core/phaseimpute/pull/51) - Update all process and fix linting errors. Remove fastqc added by the template.
+- [#56](https://github.com/nf-core/phaseimpute/pull/56) - Move to nf-test to check the output files names generated. Fix validation and concatenation by chromosomes missing. Add dedicated GLIMPSE1 subworkflow. Fix posfile generation to be done once for glimpse and stitch.
 
 ### `Fixed`
 

diff --git a/README.md b/README.md
@@ -62,7 +62,7 @@ nextflow run nf-core/phaseimpute \
    -profile <docker/singularity/.../institute> \
    --input <samplesheet.csv>  \
    --genome "GRCh38" \
-   --panel <phased_reference_panel.vcf.gz> \
+   --panel <phased_reference_panel.csv> \
    --steps "panelprep,impute" \
    --tools "glimpse1" \
    --outdir <OUTDIR>

diff --git a/conf/steps/imputation_glimpse1.config b/conf/steps/imputation_glimpse1.config
@@ -13,12 +13,17 @@
 process {
     // Configuration for the glimpse1 imputation subworkflow
 
+    // Impute with GLIMPSE1
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:.*' {
+        publishDir = [ enabled: false ]
+    }
+
     // Call the variants before imputation
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_INPUT:.*' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:BAM_GL_BCFTOOLS:.*' {
         publishDir = [ enabled: false ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_INPUT:BCFTOOLS_MPILEUP' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:BAM_GL_BCFTOOLS:BCFTOOLS_MPILEUP' {
         ext.args = [
             "-I",
             "-E",
@@ -28,53 +33,39 @@ process {
             "-Aim",
             "-C alleles"
         ].join(' ')
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.call" }
+        ext.prefix = { "${meta.id}.call" }
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_INPUT:BCFTOOLS_ANNOTATE' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:BAM_GL_BCFTOOLS:BCFTOOLS_ANNOTATE' {
         ext.args   = "--set-id '%CHROM:%POS:%REF:%ALT' -Oz"
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.annotate" }
+        ext.prefix = { "${meta.id}.annotate" }
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_INPUT:BCFTOOLS_INDEX' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:BAM_GL_BCFTOOLS:BCFTOOLS_INDEX' {
         ext.args = "--tbi"
     }
 
-    // Impute with GLIMPSE1
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:.*' {
-        publishDir = [
-            path  : { "${params.outdir}/imputation/glimpse1/" },
-            mode  : params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:GLIMPSE_CHUNK' {
-        ext.args = ["--window-size 200000", "--buffer-size 20000"].join(' ')
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.chunk" }
-        publishDir = [ enabled: false ]
-    }
-
+    // Impute the variants
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:GLIMPSE_PHASE' {
         ext.args   = ["--impute-reference-only-variants"].join(' ')
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.phase" }
+        ext.prefix = { "${meta.id}_${meta.region.replace(':','_')}_phase" }
         ext.suffix = "bcf"
         publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:BCFTOOLS_INDEX_2' {
+        ext.args   = "--tbi"
         publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:GLIMPSE_LIGATE' {
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.ligate" }
+        ext.prefix = { "${meta.id}_${meta.chr}_ligate" }
+        publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:BCFTOOLS_INDEX_2' {
-        publishDir = [
-            path: { "${params.outdir}/imputation/glimpse1" },
-            mode: params.publish_dir_mode,
-        ]
+        ext.args   = "--tbi"
+        publishDir = [ enabled: false ]
     }
 
     // Concatenate the imputed chunks

diff --git a/conf/steps/imputation_glimpse2.config b/conf/steps/imputation_glimpse2.config
@@ -13,13 +13,11 @@
 process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE2:.*' {
-        publishDir = [
-            path: { "${params.outdir}/imputation/glimpse2/" },
-            mode: params.publish_dir_mode,
-        ]
+        publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE2:GLIMPSE2_PHASE' {
+        ext.prefix   = { "${meta.id}_${meta.region.replace(':','_')}_glimpse2" }
         ext.args     = "--keep-monomorphic-ref-sites"
         ext.suffix   = "vcf.gz"
         publishDir   = [ enabled: false ]
@@ -30,6 +28,16 @@ process {
         publishDir   = [ enabled: false ]
     }
 
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE2:GLIMPSE2_LIGATE' {
+        ext.prefix = { "${meta.id}_${meta.chr}_ligate" }
+        publishDir = [ enabled: false ]
+    }
+
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE2:BCFTOOLS_INDEX_2' {
+        ext.args     = "--tbi"
+        publishDir = [ enabled: false ]
+    }
+
     // Concatenate the imputed chunks
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE2:.*' {

diff --git a/conf/steps/imputation_quilt.config b/conf/steps/imputation_quilt.config
@@ -21,7 +21,7 @@ process {
 
     // Impute quilt
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:QUILT_QUILT' {
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.impute" }
+        ext.prefix = { "${meta.id}_C${meta.chr}.impute" }
         publishDir = [enabled: false]
     }
 
@@ -33,11 +33,13 @@ process {
     // Annotate quilt imputed VCFs
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_ANNOTATE' {
         ext.args     = "--set-id '%CHROM:%POS:%REF:%ALT' -Oz"
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.impute.annotate" }
+        ext.prefix = { "${meta.id}_C${meta.chr}.impute.annotate" }
+        publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_INDEX_2' {
         ext.args     = "--tbi"
+        publishDir = [ enabled: false ]
     }
 
     // Concatenate quilt imputed VCFs

diff --git a/conf/steps/imputation_stitch.config b/conf/steps/imputation_stitch.config
@@ -11,27 +11,6 @@
 */
 
 process {
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:PREPARE_POSFILE_TSV:.*' {
-        publishDir = [
-            path: { "${params.outdir}/prep_panel/posfile/" },
-            mode: params.publish_dir_mode,
-            enabled: true
-        ]
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:PREPARE_POSFILE_TSV:BCFTOOLS_QUERY' {
-        ext.args   = "-f'%CHROM\t%POS\t%REF\t%ALT\\n'"
-        ext.prefix = { "${meta.id}_${meta.chr}_posfile_stitch" }
-        publishDir = [enabled: false]
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:PREPARE_POSFILE_TSV:GAWK' {
-        ext.args   = "'{ key = \$1 FS \$2 } !seen[key]++'" // Remove duplicates
-        ext.prefix = { "${meta.id}_${meta.chr}_posfile_stitch" }
-        ext.suffix = "txt"
-    }
-
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_STITCH:.*' {
         publishDir = [
             path: { "${params.outdir}/imputation/stitch/" },

diff --git a/conf/steps/panel_prep.config b/conf/steps/panel_prep.config
@@ -72,7 +72,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:VCFLIB_VCFFIXUP' {
-        ext.prefix   = { "${meta.panel}_${meta.chr}" }
+        ext.prefix   = { "${meta.id}_${meta.chr}" }
         publishDir   = [enabled: false]
     }
 
@@ -127,6 +127,12 @@ process {
         publishDir = [ enabled: false ]
     }
 
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SITES_EXTRACT_BCFTOOLS:GAWK' {
+        ext.args   = "'{ gsub(\",\", \"\\t\") ; key = \$1 FS \$2 } !seen[key]++'" // Remove duplicates
+        ext.prefix = { "${meta.id}_${meta.chr}_posfile_stitch" }
+        ext.suffix = "txt"
+    }
+
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SITES_EXTRACT_BCFTOOLS:TABIX_BGZIP' {
         ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites_tsv" }
         publishDir = [
@@ -142,7 +148,7 @@ process {
             "-b2",
             "-e2"
         ].join(' ')
-        ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites_tsv" }
+        ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites_tsv_gz" }
         publishDir = [
             path: { "${params.outdir}/prep_panel/sites/tsv/" },
             mode: params.publish_dir_mode,
@@ -162,6 +168,10 @@ process {
         publishDir = [ enabled: false ]
     }
 
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_PANEL:BCFTOOLS_INDEX' {
+        ext.args = "--tbi"
+    }
+
     // Subworkflow: Make chunks
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:.*' {
         publishDir = [
@@ -172,7 +182,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE_CHUNK' {
-        ext.prefix    = { "${meta.panel}_${meta.chr}_chunks_glimpse1" }
+        ext.prefix = { "${meta.id}_${meta.chr}_chunks_glimpse1" }
         publishDir = [
             path: { "${params.outdir}/prep_panel/chunks/glimpse1/" },
             mode: params.publish_dir_mode,
@@ -181,8 +191,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE2_CHUNK' {
-        ext.args = ["--window-mb 2.0"].join(' ')
-        ext.prefix    = { "${meta.panel}_${meta.chr}_chunks_glimpse2" }
+        ext.prefix    = { "${meta.id}_${meta.chr}_chunks_glimpse2" }
         publishDir = [
             path: { "${params.outdir}/prep_panel/chunks/glimpse2/" },
             mode: params.publish_dir_mode,
@@ -191,7 +200,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE2_SPLITREFERENCE' {
-        ext.prefix    = { "${meta.panel}_${meta.chr}_chunks_glimpse2" }
+        ext.prefix    = { "${meta.id}_${meta.chr}_chunks_glimpse2" }
         publishDir = [
             path: { "${params.outdir}/prep_panel/chunks/glimpse2/" },
             mode: params.publish_dir_mode,

diff --git a/conf/steps/simulation.config b/conf/steps/simulation.config
@@ -25,9 +25,24 @@ process {
         ]
     }
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_COVERAGE' {
+        publishDir = [
+            path: { "${params.outdir}/simulation/stats/" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
+        ]
         ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.stats" }
     }
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_VIEW' {
         ext.prefix = { "${meta.id}_D${meta.depth}_R${meta.region.replace(':','_')}" }
+        publishDir = [ enabled: false ]
+    }
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_INDEX_1' {
+        publishDir = [ enabled: false ]
+    }
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_MERGE' {
+        ext.prefix = { "${meta.id}" }
+    }
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_INDEX_2' {
+        ext.args = ""
     }
 }
diff --git a/conf/steps/validation.config b/conf/steps/validation.config
@@ -17,6 +17,7 @@ process {
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:.*' {
         publishDir = [ enabled: false ]
     }
+
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:BCFTOOLS_MPILEUP' {
         ext.args = [
             "-I",
@@ -27,18 +28,19 @@ process {
             "-Aim",
             "-C alleles"
         ].join(' ')
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}_truth.call" }
+        ext.prefix = { "${meta.id}_truth.call" }
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:BCFTOOLS_ANNOTATE' {
         ext.args   = ["--set-id '%CHROM:%POS:%REF:%ALT'", "-Oz"].join(' ')
-        ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}.annotate" }
+        ext.prefix = { "${meta.id}_${meta.chr}.annotate" }
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:BCFTOOLS_INDEX' {
         ext.args = "--tbi"
     }
 
+    // Concatenate the truth set
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_TRUTH:.*' {
         ext.prefix = { "${meta.id}_truth_concat" }
         publishDir = [
@@ -67,21 +69,21 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CONCORDANCE_GLIMPSE2:GLIMPSE2_CONCORDANCE' {
         ext.args   = "--out-r2-per-site"
-        ext.prefix = { "${meta.id}.concordance" }
+        ext.prefix = { "${meta.id}_P${meta.panel}_T${meta.tools}.concordance" }
         publishDir = [ enabled: false ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CONCORDANCE_GLIMPSE2:GAWK' {
-        ext.args   = "'(NR == 1) || (FNR > 1)'" // Skip header line
-        ext.suffix = { "txt" }
-    }
-
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CONCORDANCE_GLIMPSE2:GUNZIP' {
+        ext.prefix = { "${meta.id}_P${meta.panel}_T${meta.tools}" }
         publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CONCORDANCE_GLIMPSE2:ADD_COLUMNS' {
-        ext.prefix = { "${meta.id}_D${meta.depth}_P${meta.panel}_SNP" }
-        publishDir = [ enabled: false ]
+        ext.prefix = { "${meta.id}_P${meta.panel}_T${meta.tools}_SNP" }
+    }
+
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CONCORDANCE_GLIMPSE2:GAWK' {
+        ext.args   = "'(NR == 1) || (FNR > 1)'" // Skip header line
+        ext.suffix = { "txt" }
     }
 }
diff --git a/conf/test.config b/conf/test.config
@@ -34,3 +34,7 @@ params {
     // Impute tools
     tools  = "glimpse1"
 }
+
+withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE_CHUNK' {
+    ext.args   = ["--window-size 10000", "--window-count 400", "--buffer-size 5000", "--buffer-count 30"].join(' ')
+}