nf-cmgg · nvnieuwk · Nov 12, 2024 · Oct 23, 2024 · Oct 23, 2024 · Oct 23, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,14 +11,15 @@ on:
 
 env:
   NXF_ANSI_LOG: false
+  NFT_MAX_SHARDS: 5
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
   test_all:
-    name: Run nf-test with ${{ matrix.test }}-${{ matrix.NXF_VER }}
+    name: Run ${{ matrix.filter }} tests | shard ${{ matrix.shard }} (${{ matrix.NXF_VER }})
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-cmgg/germline') }}"
     runs-on: ubuntu-latest
@@ -27,21 +28,11 @@ jobs:
         NXF_VER:
           - "24.04.2"
           - "latest-everything"
-        test:
-          - "pipeline_default"
-          - "pipeline_callers"
-          - "pipeline_variations"
-          - "pipeline_variations2"
-          - "pipeline_gvcfs"
-          - "cram_call_genotype_gatk4"
-          - "cram_call_vardictjava"
-          - "cram_prepare_samtools_bedtools"
-          - "input_split_bedtools"
-          - "vcf_annotation"
-          - "vcf_extract_relate_somalier"
-          - "vcf_ped_rtgtools"
-          - "vcf_upd_updio"
-          - "vcf_validate_small_variants"
+        filter:
+          - "process"
+          - "workflow"
+          - "pipeline"
+        shard: [1, 2, 3, 4, 5]
     steps:
       - name: Free some space
         run: |
@@ -52,6 +43,8 @@ jobs:
 
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          fetch-depth: 0
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v2
@@ -65,9 +58,14 @@ jobs:
         run: |
           conda install -c bioconda nf-test
 
-      - name: Run pipeline with test data
+      - name: "Run ${{ matrix.filter }} tests | ${{ matrix.shard }}/${{ env.NFT_MAX_SHARDS }}"
         run: |
-          $CONDA/bin/nf-test test --tag ${{ matrix.test }} --junitxml=default.xml
+          $CONDA/bin/nf-test test \
+            --ci \
+            --changed-since HEAD^ \
+            --shard ${{ matrix.shard }}/${{ env.NFT_MAX_SHARDS }} \
+            --filter ${{ matrix.filter }} \
+            --junitxml=default.xml
 
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v3

diff --git a/conf/modules.config b/conf/modules.config
@@ -124,15 +124,15 @@ process {
 
     /*
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-        GATK4_HAPLOTYPCECALLER
+        GATK4 HAPLOTYPCECALLER
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     */
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:CRAM_CALL_GATK4:GATK4_CALIBRATEDRAGSTRMODEL\$" {
+    withName: "^.*CRAM_CALL_GATK4:GATK4_CALIBRATEDRAGSTRMODEL\$" {
         ext.args    = "--parallel"
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:CRAM_CALL_GATK4:GATK4_HAPLOTYPECALLER\$" {
+    withName: "^.*CRAM_CALL_GATK4:GATK4_HAPLOTYPECALLER\$" {
         time        = { 16.h  * task.attempt }
         ext.prefix  = {"${meta.id}.g"}
         ext.args    = {
@@ -169,7 +169,7 @@ process {
         ext.args   = ''
     }
 
-    withName: "^.*CRAM_CALL_GATK4:BCFTOOLS_STATS_SINGLE\$" {
+    withName: "^.*CRAM_CALL_GATK4:BCFTOOLS_STATS\$" {
         publishDir = [
             overwrite: true,
             enabled: true,
@@ -180,12 +180,63 @@ process {
         ext.prefix = final_prefix
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:GVCF_JOINT_GENOTYPE_GATK4:BCFTOOLS_QUERY\$" {
+    /*
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        ELPREP
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    */
+
+    withName: "^.*BAM_CALL_ELPREP:ELPREP_FILTER\$" {
+        cpus    = { 25 * task.attempt }
+        memory  = { 250.GB * task.attempt }
+        ext.args = "--reference-confidence GVCF"
+    }
+
+    withName: "^.*BAM_CALL_ELPREP:VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" {
+        publishDir  = [
+            overwrite: true,
+            enabled: true,
+            mode: params.publish_dir_mode,
+            path: individual_output,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ] // SAVE
+        ext.prefix = { "${meta.id}.${meta.caller}.g" }
+        ext.args   = '--allow-overlaps --output-type z'
+    }
+
+    withName: "^.*BAM_CALL_ELPREP:VCF_CONCAT_BCFTOOLS:TABIX_TABIX\$" {
+        publishDir = [
+            overwrite: true,
+            enabled: true,
+            path: individual_output,
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ] // SAVE
+    }
+
+    withName: "^.*BAM_CALL_ELPREP:BCFTOOLS_STATS\$" {
+        publishDir = [
+            overwrite: true,
+            enabled: true,
+            path: individual_reports,
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ] // SAVE
+        ext.prefix = final_prefix
+    }
+
+    /*
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        GVCF JOINT GENOTYPING
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    */
+
+    withName: "^.*GVCF_JOINT_GENOTYPE_GATK4:BCFTOOLS_QUERY\$" {
         ext.args = "--exclude 'QUAL=\".\"' --format '%CHROM\t%POS0\t%END\\n'"
         ext.suffix = "bed"
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:GVCF_JOINT_GENOTYPE_GATK4:MERGE_BEDS\$" {
+    withName: "^.*GVCF_JOINT_GENOTYPE_GATK4:MERGE_BEDS\$" {
         ext.args = "-d ${params.merge_distance}"
         publishDir = [
             enabled: true,
@@ -196,12 +247,12 @@ process {
         ] // SAVE
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:GVCF_JOINT_GENOTYPE_GATK4:GAWK\$" {
+    withName: "^.*GVCF_JOINT_GENOTYPE_GATK4:GAWK\$" {
         ext.args2 = '\'BEGIN {FS="\t"}; {print \$1 FS "0" FS \$2}\''
         ext.suffix = "bed"
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:GVCF_JOINT_GENOTYPE_GATK4:GATK4_GENOMICSDBIMPORT\$" {
+    withName: "^.*GVCF_JOINT_GENOTYPE_GATK4:GATK4_GENOMICSDBIMPORT\$" {
         label = { meta.family_samples.tokenize(",").size() <= 10 ? "process_medium" : "process_high" }
         time = { 16.h  * task.attempt }
         // Lots of parameters are fetched from https://gatk.broadinstitute.org/hc/en-us/articles/360056138571-GenomicsDBImport-usage-and-performance-guidelines
@@ -228,7 +279,7 @@ process {
         ] // SAVE
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:GVCF_JOINT_GENOTYPE_GATK4:GATK4_GENOTYPEGVCFS\$" {
+    withName: "^.*GVCF_JOINT_GENOTYPE_GATK4:GATK4_GENOTYPEGVCFS\$" {
         time = { 16.h  * task.attempt }
         ext.args = {
             [
@@ -251,30 +302,13 @@ process {
         ] // SAVE
     }
 
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:VCF_FILTER_BCFTOOLS:FILTER_1\$" {
-        ext.prefix = { "${meta.id}_filtered_snps" }
-        ext.args = {"--output-type z --soft-filter 'GATKCutoffSNP' -e 'TYPE=\"snp\" && (MQRankSum < -12.5 || ReadPosRankSum < -8.0 || QD < 2.0 || FS > 60.0 || MQ < 30.0)' -m '+'"}
-    }
-
-    withName: "^.*CRAM_CALL_GENOTYPE_GATK4:VCF_FILTER_BCFTOOLS:FILTER_2\$" {
-        ext.prefix = enableOutput("filter") ? final_prefix : {"${meta.id}.filtered"}
-        ext.args = {'--output-type z --soft-filter \'GATKCutoffIndel\' -e \'TYPE="indel" && (ReadPosRankSum < -20.0 || QD < 2.0 || FS > 200.0 || SOR > 10.0 )\' -m \'+\''}
-        publishDir = [
-            enabled: enableOutput("filter"),
-            overwrite: true,
-            path: final_output,
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ] // SAVE
-    }
-
     /*
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         VARDICTJAVA
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     */
 
-    withName: "^.*CRAM_CALL_VARDICTJAVA:VARDICTJAVA\$" {
+    withName: "^.*BAM_CALL_VARDICTJAVA:VARDICTJAVA\$" {
         time        = { 16.h  * task.attempt }
         ext.prefix  = {"${meta.id}"}
         ext.args    = {
@@ -291,7 +325,7 @@ process {
         }
     }
 
-    withName: "^.*CRAM_CALL_VARDICTJAVA:VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" {
+    withName: "^.*BAM_CALL_VARDICTJAVA:VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" {
         ext.args   = '--allow-overlaps --output-type z'
         ext.prefix = enableOutput("original") ? final_prefix : {"${meta.id}.concat"}
         publishDir  = [
@@ -303,7 +337,7 @@ process {
         ] // SAVE
     }
 
-    withName: "^.*CRAM_CALL_VARDICTJAVA:TABIX_VCFANNO\$" {
+    withName: "^.*BAM_CALL_VARDICTJAVA:TABIX_VCFANNO\$" {
         ext.prefix = enableOutput("original") ? final_prefix : {"${meta.id}.vcfanno"}
         publishDir  = [
             overwrite: true,
@@ -314,27 +348,50 @@ process {
         ] // SAVE
     }
 
-    withName: "^.*CRAM_CALL_VARDICTJAVA:VCF_FILTER_BCFTOOLS:FILTER_1\$" {
-        ext.prefix = { "${meta.id}.filtered1" }
-        ext.args = "-i 'QUAL >= 0${params.only_pass ? " && FILTER=\"PASS\"" : ""}' --output-type z"
-    }
-
-    withName: "^.*CRAM_CALL_VARDICTJAVA:VCF_FILTER_BCFTOOLS:FILTER_2\$" {
-        ext.args = "--soft-filter 'LowFreqBias' --mode '+' -e  'FORMAT/AF[0:*] < 0.02 && FORMAT/VD[0] < 30 && INFO/SBF < 0.1 && INFO/NM >= 2.0' --output-type z"
-        ext.prefix = enableOutput("filter") ? final_prefix : {"${meta.id}.filtered"}
+    withName: "^.*BAM_CALL_VARDICTJAVA:TABIX_TABIX\$" {
         publishDir  = [
             overwrite: true,
-            enabled: enableOutput("filter"),
+            enabled: enableOutput("filter") || enableOutput("original"),
             mode: params.publish_dir_mode,
             path: final_output,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ] // SAVE
     }
 
-    withName: "^.*CRAM_CALL_VARDICTJAVA:TABIX_TABIX\$" {
+    /*
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        FILTER
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    */
+
+    withName: "^.*VCF_FILTER_BCFTOOLS:FILTER_1\$" {
+        ext.prefix = { "${meta.id}.filtered1" }
+        ext.args = {
+            meta.caller == "vardict" ?
+                "-i 'QUAL >= 0${params.only_pass ? " && FILTER=\"PASS\"" : ""}' --output-type z":
+            meta.caller == "haplotypecaller" ?
+                "--output-type z --soft-filter 'GATKCutoffSNP' -e 'TYPE=\"snp\" && (MQRankSum < -12.5 || ReadPosRankSum < -8.0 || QD < 2.0 || FS > 60.0 || MQ < 30.0)' -m '+'":
+            meta.caller == "elprep" ?
+                "--output-type z --soft-filter 'GATKCutoffSNP' -e 'TYPE=\"snp\" && (MQRankSum < -12.5 || ReadPosRankSum < -8.0 || QD < 2.0 || FS > 60.0 || MQ < 30.0)' -m '+'":
+            ""
+        }
+    }
+
+    withName: "^.*VCF_FILTER_BCFTOOLS:FILTER_2\$" {
+        ext.args = {
+            meta.caller == "vardict" ?
+                "--soft-filter 'LowFreqBias' --mode '+' -e  'FORMAT/AF[0:*] < 0.02 && FORMAT/VD[0] < 30 && INFO/SBF < 0.1 && INFO/NM >= 2.0' --output-type z" :
+            meta.caller == "haplotypecaller" ?
+                '--output-type z --soft-filter \'GATKCutoffIndel\' -e \'TYPE="indel" && (ReadPosRankSum < -20.0 || QD < 2.0 || FS > 200.0 || SOR > 10.0 )\' -m \'+\'' :
+            meta.caller == "elprep" ?
+                '--output-type z --soft-filter \'GATKCutoffIndel\' -e \'TYPE="indel" && (ReadPosRankSum < -20.0 || QD < 2.0 || FS > 200.0 || SOR > 10.0 )\' -m \'+\'' :
+            ""
+
+        }
+        ext.prefix = enableOutput("filter") ? final_prefix : {"${meta.id}.filtered"}
         publishDir  = [
             overwrite: true,
-            enabled: enableOutput("filter") || enableOutput("original"),
+            enabled: enableOutput("filter"),
             mode: params.publish_dir_mode,
             path: final_output,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }

diff --git a/conf/test.config b/conf/test.config
@@ -34,6 +34,7 @@ params {
     fasta           = "https://github.com/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/genome/hg38_chr21_22000000_23000000.fasta"
     fai             = "https://github.com/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/genome/hg38_chr21_22000000_23000000.fasta.fai"
     dict            = "https://github.com/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/genome/hg38_chr21_22000000_23000000.dict"
+    elfasta         = "https://github.com/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/genome/hg38_chr21_22000000_23000000.elfasta"
     sdf             = "https://github.com/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/genome/hg38_chr21_22000000_23000000_sdf.tar.gz"
     strtablefile    = "https://github.com/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/genome/hg38_chr21_22000000_23000000.strtable.zip"
 

diff --git a/lib/GlobalVariables.groovy b/lib/GlobalVariables.groovy
@@ -4,9 +4,11 @@ import java.nio.file.Path
 
 class GlobalVariables {
     // The available callers
-    public static List availableCallers = ["haplotypecaller", "vardict"]
+    public static List availableCallers = ["haplotypecaller", "vardict", "elprep"]
 
-    public static List gvcfCallers = ["haplotypecaller"]
+    public static List gvcfCallers = ["haplotypecaller", "elprep"]
+
+    public static List bamCallers = ["elprep", "vardict"]
 
     public static Map<String,Path> pedFiles = [:]
 

diff --git a/main.nf b/main.nf
@@ -19,6 +19,7 @@ include { getGenomeAttribute } from './subworkflows/local/utils_cmgg_germline_pi
 params.fasta                = getGenomeAttribute('fasta', params.genomes, params.genome)
 params.fai                  = getGenomeAttribute('fai', params.genomes, params.genome)
 params.dict                 = getGenomeAttribute('dict', params.genomes, params.genome)
+params.elfasta              = getGenomeAttribute('elfasta', params.genomes, params.genome)
 params.strtablefile         = getGenomeAttribute('strtablefile', params.genomes, params.genome)
 params.sdf                  = getGenomeAttribute('sdf', params.genomes, params.genome)
 params.dbsnp                = getGenomeAttribute('dbsnp', params.genomes, params.genome)
@@ -73,6 +74,7 @@ workflow NFCMGG_GERMLINE {
         pipeline_params.fasta,
         pipeline_params.fai,
         pipeline_params.dict,
+        pipeline_params.elfasta,
         pipeline_params.strtablefile,
         pipeline_params.sdf,
         pipeline_params.dbsnp,
@@ -103,6 +105,7 @@ workflow NFCMGG_GERMLINE {
         pipeline_params.automap_panel,
         pipeline_params.outdir,
         GlobalVariables.pedFiles,
+        pipeline_params.elsites,
 
         // Boolean inputs
         pipeline_params.dragstr,

diff --git a/modules.json b/modules.json
@@ -66,6 +66,17 @@
                         "git_sha": "cb08035150685b11d890d90c9534d4f16869eaec",
                         "installed_by": ["modules"]
                     },
+                    "elprep/fastatoelfasta": {
+                        "branch": "master",
+                        "git_sha": "74ac5351a11a184171489dee73652e8b69ba9d22",
+                        "installed_by": ["modules"]
+                    },
+                    "elprep/filter": {
+                        "branch": "master",
+                        "git_sha": "909c4dcdbb1e751214e2bb155e8c0a59633ed12a",
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/elprep/filter/elprep-filter.diff"
+                    },
                     "ensemblvep/download": {
                         "branch": "master",
                         "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
@@ -183,7 +194,7 @@
                     "tabix/bgzip": {
                         "branch": "master",
                         "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
-                        "installed_by": ["modules"]
+                        "installed_by": ["vcf_annotate_ensemblvep_snpeff"]
                     },
                     "tabix/bgziptabix": {
                         "branch": "master",

diff --git a/modules/nf-core/elprep/fastatoelfasta/environment.yml b/modules/nf-core/elprep/fastatoelfasta/environment.yml