From e79b306d9332c41b919fdc16f7d9486ffb967183 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 11:45:56 +0200 Subject: [PATCH 01/47] FIX: gatk4/createsequencedictionary default memory --- CHANGELOG.md | 11 +++++++++++ modules.json | 6 +++--- .../nf-core/gatk4/createsequencedictionary/main.nf | 2 +- .../nf-core/gatk4/filtervarianttranches/main.nf | 2 +- .../nf-core/gatk4/filtervarianttranches/meta.yml | 14 ++++++++------ modules/nf-core/gatk4/markduplicatesspark/main.nf | 1 - nextflow.config | 2 +- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6e98f019..78eb11983b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## dev + +### Changed + +- [#1086](https://github.com/nf-core/sarek/pull/1086) - Back to dev +- [#1086](https://github.com/nf-core/sarek/pull/1086) - Minor modules update + +### Fixed + +- [#1086](https://github.com/nf-core/sarek/pull/1086) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) + ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure Pierikjaure is a previous spelling of Bierikjávrre. diff --git a/modules.json b/modules.json index 3e59e23517..97eb570c13 100644 --- a/modules.json +++ b/modules.json @@ -198,7 +198,7 @@ }, "gatk4/createsequencedictionary": { "branch": "master", - "git_sha": "2df2a11d5b12f2a73bca74f103691bc35d83c5fd", + "git_sha": "541811d779026c5d395925895fa5ed35e7216cc0", "installed_by": ["modules"] }, "gatk4/estimatelibrarycomplexity": { @@ -213,7 +213,7 @@ }, "gatk4/filtervarianttranches": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "541811d779026c5d395925895fa5ed35e7216cc0", "installed_by": ["modules"] }, "gatk4/gatherbqsrreports": { @@ -263,7 +263,7 @@ }, "gatk4/markduplicatesspark": { "branch": "master", - "git_sha": "4b7d4863a5883b76e6bff13b6e52468fab090c5b", + "git_sha": "541811d779026c5d395925895fa5ed35e7216cc0", "installed_by": ["modules"] }, "gatk4/mergemutectstats": { diff --git a/modules/nf-core/gatk4/createsequencedictionary/main.nf b/modules/nf-core/gatk4/createsequencedictionary/main.nf index 41abb235c6..15a86beab3 100644 --- a/modules/nf-core/gatk4/createsequencedictionary/main.nf +++ b/modules/nf-core/gatk4/createsequencedictionary/main.nf @@ -20,7 +20,7 @@ process GATK4_CREATESEQUENCEDICTIONARY { script: def args = task.ext.args ?: '' - def avail_mem = 6 + def avail_mem = 6144 if (!task.memory) { log.info '[GATK CreateSequenceDictionary] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' } else { diff --git a/modules/nf-core/gatk4/filtervarianttranches/main.nf b/modules/nf-core/gatk4/filtervarianttranches/main.nf index f72e64cc7d..be232ab4bb 100644 --- a/modules/nf-core/gatk4/filtervarianttranches/main.nf +++ b/modules/nf-core/gatk4/filtervarianttranches/main.nf @@ -27,8 +27,8 @@ process GATK4_FILTERVARIANTTRANCHES { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def resources = resources.collect{"--resource $it"}.join(' ') + def avail_mem = 3072 if (!task.memory) { log.info '[GATK FilterVariantTranches] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' diff --git a/modules/nf-core/gatk4/filtervarianttranches/meta.yml b/modules/nf-core/gatk4/filtervarianttranches/meta.yml index 72cf9f319f..4152a65c1f 100644 --- a/modules/nf-core/gatk4/filtervarianttranches/meta.yml +++ b/modules/nf-core/gatk4/filtervarianttranches/meta.yml @@ -3,15 +3,17 @@ description: Apply tranche filtering keywords: - gatk4 - filtervarianttranches - + - tranche_filtering tools: - "gatk4": - description: Genome Analysis Toolkit (GATK4) + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. homepage: https://gatk.broadinstitute.org/hc/en-us - documentation: https://gatk.broadinstitute.org/hc/en-us - tool_dev_url: https://github.com/broadinstitute/gatk - doi: "10.1158/1538-7445.AM2017-3590" - licence: ["BSD-3-clause"] + documentation: https://gatk.broadinstitute.org/hc/en-us/articles/360051308071-FilterVariantTranches + doi: 10.1158/1538-7445.AM2017-3590 + licence: ["Apache-2.0"] input: - meta: diff --git a/modules/nf-core/gatk4/markduplicatesspark/main.nf b/modules/nf-core/gatk4/markduplicatesspark/main.nf index f64fbfd521..f51d598f86 100644 --- a/modules/nf-core/gatk4/markduplicatesspark/main.nf +++ b/modules/nf-core/gatk4/markduplicatesspark/main.nf @@ -25,7 +25,6 @@ process GATK4_MARKDUPLICATES_SPARK { prefix = task.ext.prefix ?: "${meta.id}" def input_list = bam.collect{"--input $it"}.join(' ') - def avail_mem = 3072 if (!task.memory) { log.info '[GATK MarkDuplicatesSpark] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' diff --git a/nextflow.config b/nextflow.config index 8d0dd50a29..dafabcfca1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -323,7 +323,7 @@ manifest { description = """An open-source analysis pipeline to detect germline or somatic variants from whole genome or targeted sequencing""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '3.2.1' + version = '3.3dev' doi = '10.12688/f1000research.16665.2, 10.5281/zenodo.4063683' } From 7cf95fe8b814f83bc6c5ea11124b3ca9ceb4f8f1 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 11:47:03 +0200 Subject: [PATCH 02/47] update CHANGELOG --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78eb11983b..c3e1412a8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- [#1086](https://github.com/nf-core/sarek/pull/1086) - Back to dev -- [#1086](https://github.com/nf-core/sarek/pull/1086) - Minor modules update +- [#1087](https://github.com/nf-core/sarek/pull/1087) - Back to dev +- [#1087](https://github.com/nf-core/sarek/pull/1087) - Minor modules update ### Fixed -- [#1086](https://github.com/nf-core/sarek/pull/1086) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) +- [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure From ecf8e727042a08a5d8b1e6cad657e38f0f67225d Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 12:02:37 +0200 Subject: [PATCH 03/47] remove md5checksum --- tests/test_joint_germline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_joint_germline.yml b/tests/test_joint_germline.yml index 2b6821f9fe..dc0b60fa36 100644 --- a/tests/test_joint_germline.yml +++ b/tests/test_joint_germline.yml @@ -13,13 +13,13 @@ - path: results/preprocessing/recalibrated/test/test.recal.cram.crai should_exist: false - path: results/reports/bcftools/haplotypecaller/joint_variant_calling/joint_germline.bcftools_stats.txt - md5sum: 404d1208df301a6726f5543245625ef3 + # Not stable enough - path: results/reports/vcftools/haplotypecaller/joint_variant_calling/joint_germline.FILTER.summary - md5sum: 87a84b5f8ac3d3cbeeef7d60afcdbfe7 + # Not stable enough - path: results/reports/vcftools/haplotypecaller/joint_variant_calling/joint_germline.TsTv.count - md5sum: 974f6922981c87bb017b124aa009f654 + # Not stable enough - path: results/reports/vcftools/haplotypecaller/joint_variant_calling/joint_germline.TsTv.qual - md5sum: 1e34357e5848c318f8c2c7d3b041d229 + # Not stable enough - path: results/variant_calling/haplotypecaller/joint_variant_calling/joint_germline.vcf.gz # binary changes md5sums on reruns - path: results/variant_calling/haplotypecaller/joint_variant_calling/joint_germline.vcf.gz.tbi From 01119fe313386010717c583afb2204beb131d309 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 12:37:01 +0200 Subject: [PATCH 04/47] replace test by test_cache and add test with non obfuscated files --- conf/test.config | 114 ++++++++++++++++++ ...fastq.config => alignment_to_fastq.config} | 0 ...st_annotation.config => annotation.config} | 0 conf/test/{test.config => cache.config} | 0 ...s_bam.config => markduplicates_bam.config} | 0 ...cram.config => markduplicates_cram.config} | 0 ...o_intervals.config => no_intervals.config} | 0 conf/test/{test_pair.config => pair.config} | 0 ...onfig => prepare_recalibration_bam.config} | 0 ...nfig => prepare_recalibration_cram.config} | 0 ...rate_bam.config => recalibrate_bam.config} | 0 ...te_cram.config => recalibrate_cram.config} | 0 ...m_mapped.config => save_bam_mapped.config} | 0 ...test_skip_bqsr.config => skip_bqsr.config} | 0 ...ates.config => skip_markduplicates.config} | 0 ..._split_fastq.config => split_fastq.config} | 0 .../{test_targeted.config => targeted.config} | 0 conf/test/{test_tools.config => tools.config} | 0 ..._germline.config => tools_germline.config} | 0 ...ls_somatic.config => tools_somatic.config} | 0 ...scat.config => tools_somatic_ascat.config} | 0 ...umoronly.config => tools_tumoronly.config} | 0 .../{test_trimming.config => trimming.config} | 0 conf/test/{test_umi.config => umi.config} | 0 ...atk_spark.config => use_gatk_spark.config} | 0 ....config => variantcalling_channels.config} | 0 conf/{test => }/test_full.config | 0 conf/{test => }/test_full_germline.config | 0 nextflow.config | 55 ++++----- tests/test_aligner_bwamem.yml | 4 +- tests/test_aligner_bwamem2.yml | 4 +- tests/test_aligner_dragmap.yml | 4 +- tests/test_alignment_to_fastq.yml | 2 +- tests/test_annotation_cache.yml | 4 +- tests/test_annotation_merge.yml | 4 +- tests/test_annotation_snpeff.yml | 2 +- tests/test_annotation_vep.yml | 4 +- tests/test_cnvkit.yml | 8 +- tests/test_concat_germline_vcfs.yml | 2 +- tests/test_controlfreec.yml | 6 +- tests/test_deepvariant.yml | 4 +- tests/test_fastp.yml | 4 +- tests/test_freebayes.yml | 12 +- tests/test_gatk4_spark.yml | 6 +- tests/test_haplotypecaller.yml | 4 +- tests/test_haplotypecaller_skip_filter.yml | 4 +- tests/test_intervals.yml | 6 +- tests/test_joint_germline.yml | 2 +- tests/test_manta.yml | 12 +- tests/test_markduplicates_from_bam.yml | 4 +- tests/test_markduplicates_from_cram.yml | 4 +- tests/test_mpileup.yml | 8 +- tests/test_msisensorpro.yml | 4 +- tests/test_mutect2.yml | 4 +- tests/test_prepare_recalibration_from_bam.yml | 4 +- .../test_prepare_recalibration_from_cram.yml | 4 +- tests/test_recalibrate_from_bam.yml | 4 +- tests/test_recalibrate_from_cram.yml | 4 +- tests/test_samplesheet_validation_spaces.yml | 2 +- tests/test_save_mapped.yml | 2 +- tests/test_save_output_as_bam_only.yml | 2 +- tests/test_skip_all_qc.yml | 2 +- tests/test_skip_markduplicates.yml | 6 +- tests/test_strelka.yml | 14 +-- tests/test_strelka_bp.yml | 4 +- tests/test_tiddit.yml | 6 +- tests/test_tools_manually.yml | 6 +- tests/test_tumor_normal_pair.yml | 2 +- tests/test_umi.yml | 4 +- 69 files changed, 236 insertions(+), 121 deletions(-) create mode 100644 conf/test.config rename conf/test/{test_alignment_to_fastq.config => alignment_to_fastq.config} (100%) rename conf/test/{test_annotation.config => annotation.config} (100%) rename conf/test/{test.config => cache.config} (100%) rename conf/test/{test_markduplicates_bam.config => markduplicates_bam.config} (100%) rename conf/test/{test_markduplicates_cram.config => markduplicates_cram.config} (100%) rename conf/test/{test_no_intervals.config => no_intervals.config} (100%) rename conf/test/{test_pair.config => pair.config} (100%) rename conf/test/{test_prepare_recalibration_bam.config => prepare_recalibration_bam.config} (100%) rename conf/test/{test_prepare_recalibration_cram.config => prepare_recalibration_cram.config} (100%) rename conf/test/{test_recalibrate_bam.config => recalibrate_bam.config} (100%) rename conf/test/{test_recalibrate_cram.config => recalibrate_cram.config} (100%) rename conf/test/{test_save_bam_mapped.config => save_bam_mapped.config} (100%) rename conf/test/{test_skip_bqsr.config => skip_bqsr.config} (100%) rename conf/test/{test_skip_markduplicates.config => skip_markduplicates.config} (100%) rename conf/test/{test_split_fastq.config => split_fastq.config} (100%) rename conf/test/{test_targeted.config => targeted.config} (100%) rename conf/test/{test_tools.config => tools.config} (100%) rename conf/test/{test_tools_germline.config => tools_germline.config} (100%) rename conf/test/{test_tools_somatic.config => tools_somatic.config} (100%) rename conf/test/{test_tools_somatic_ascat.config => tools_somatic_ascat.config} (100%) rename conf/test/{test_tools_tumoronly.config => tools_tumoronly.config} (100%) rename conf/test/{test_trimming.config => trimming.config} (100%) rename conf/test/{test_umi.config => umi.config} (100%) rename conf/test/{test_use_gatk_spark.config => use_gatk_spark.config} (100%) rename conf/test/{test_variantcalling_channels.config => variantcalling_channels.config} (100%) rename conf/{test => }/test_full.config (100%) rename conf/{test => }/test_full_germline.config (100%) diff --git a/conf/test.config b/conf/test.config new file mode 100644 index 0000000000..ced52f5555 --- /dev/null +++ b/conf/test.config @@ -0,0 +1,114 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/sarek -profile test,, --outdir +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.5GB' + max_time = '8.h' + + // Input data + input = "${projectDir}/tests/csv/3.0/fastq_single.csv" + + // Small reference genome + genome = null + igenomes_ignore = true + dbsnp = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta" + germline_resource = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz" + intervals = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.interval_list" + known_indels = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz" + snpeff_db = 105 + snpeff_genome = 'WBcel235' + snpeff_version = '5.1' + vep_cache_version = 106 + vep_genome = 'WBcel235' + vep_species = 'caenorhabditis_elegans' + vep_version = '106.1' + + // default params + split_fastq = 0 // no FASTQ splitting + tools = 'strelka' // Variant calling with Strelka + + // Ignore params that will throw warning through params validation + schema_ignore_params = 'genomes,snpeff_version,vep_version' +} + +process { + withName:'.*:FREEC_SOMATIC'{ + ext.args = { + [ + "sample":[ + inputformat: "pileup", + mateorientation: "FR" + ], + "general":[ + bedgraphoutput: "TRUE", + noisydata: "TRUE", + minexpectedgc: "0", + readcountthreshold: "1", + sex: meta.sex, + window: "10", + ], + "control":[ + inputformat: "pileup", + mateorientation: "FR" + ] + ] + } + } + + if (params.tools && params.tools.split(',').contains('mutect2')) { + withName: '.*MUTECT2_PAIRED'{ + //sample name from when the test data was generated + ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " } + } + } + + withName: '.*:FILTERVARIANTTRANCHES'{ + ext.args = { "--info-key CNN_1D --indel-tranche 0" } + } +} + +// Enable container engines/virtualisation envs for CI testing +// only works when specified with the profile ENV +// otherwise tests can be done with the regular provided profiles +if (System.getenv('PROFILE')) { + if ("$PROFILE" == "conda") { + conda.createTimeout = "120 min" + conda.enabled = true + charliecloud.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + singularity.enabled = false + } else if ("$PROFILE" == "docker") { + conda.enabled = false + docker.enabled = true + docker.userEmulation = { params.use_gatk_spark ? false : true }.call() + charliecloud.enabled = false + podman.enabled = false + shifter.enabled = false + singularity.enabled = false + } else if ("$PROFILE" == "singularity") { + conda.enabled = false + params.singularity_pull_docker_container = false + singularity.autoMounts = true + singularity.enabled = true + charliecloud.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + } +} diff --git a/conf/test/test_alignment_to_fastq.config b/conf/test/alignment_to_fastq.config similarity index 100% rename from conf/test/test_alignment_to_fastq.config rename to conf/test/alignment_to_fastq.config diff --git a/conf/test/test_annotation.config b/conf/test/annotation.config similarity index 100% rename from conf/test/test_annotation.config rename to conf/test/annotation.config diff --git a/conf/test/test.config b/conf/test/cache.config similarity index 100% rename from conf/test/test.config rename to conf/test/cache.config diff --git a/conf/test/test_markduplicates_bam.config b/conf/test/markduplicates_bam.config similarity index 100% rename from conf/test/test_markduplicates_bam.config rename to conf/test/markduplicates_bam.config diff --git a/conf/test/test_markduplicates_cram.config b/conf/test/markduplicates_cram.config similarity index 100% rename from conf/test/test_markduplicates_cram.config rename to conf/test/markduplicates_cram.config diff --git a/conf/test/test_no_intervals.config b/conf/test/no_intervals.config similarity index 100% rename from conf/test/test_no_intervals.config rename to conf/test/no_intervals.config diff --git a/conf/test/test_pair.config b/conf/test/pair.config similarity index 100% rename from conf/test/test_pair.config rename to conf/test/pair.config diff --git a/conf/test/test_prepare_recalibration_bam.config b/conf/test/prepare_recalibration_bam.config similarity index 100% rename from conf/test/test_prepare_recalibration_bam.config rename to conf/test/prepare_recalibration_bam.config diff --git a/conf/test/test_prepare_recalibration_cram.config b/conf/test/prepare_recalibration_cram.config similarity index 100% rename from conf/test/test_prepare_recalibration_cram.config rename to conf/test/prepare_recalibration_cram.config diff --git a/conf/test/test_recalibrate_bam.config b/conf/test/recalibrate_bam.config similarity index 100% rename from conf/test/test_recalibrate_bam.config rename to conf/test/recalibrate_bam.config diff --git a/conf/test/test_recalibrate_cram.config b/conf/test/recalibrate_cram.config similarity index 100% rename from conf/test/test_recalibrate_cram.config rename to conf/test/recalibrate_cram.config diff --git a/conf/test/test_save_bam_mapped.config b/conf/test/save_bam_mapped.config similarity index 100% rename from conf/test/test_save_bam_mapped.config rename to conf/test/save_bam_mapped.config diff --git a/conf/test/test_skip_bqsr.config b/conf/test/skip_bqsr.config similarity index 100% rename from conf/test/test_skip_bqsr.config rename to conf/test/skip_bqsr.config diff --git a/conf/test/test_skip_markduplicates.config b/conf/test/skip_markduplicates.config similarity index 100% rename from conf/test/test_skip_markduplicates.config rename to conf/test/skip_markduplicates.config diff --git a/conf/test/test_split_fastq.config b/conf/test/split_fastq.config similarity index 100% rename from conf/test/test_split_fastq.config rename to conf/test/split_fastq.config diff --git a/conf/test/test_targeted.config b/conf/test/targeted.config similarity index 100% rename from conf/test/test_targeted.config rename to conf/test/targeted.config diff --git a/conf/test/test_tools.config b/conf/test/tools.config similarity index 100% rename from conf/test/test_tools.config rename to conf/test/tools.config diff --git a/conf/test/test_tools_germline.config b/conf/test/tools_germline.config similarity index 100% rename from conf/test/test_tools_germline.config rename to conf/test/tools_germline.config diff --git a/conf/test/test_tools_somatic.config b/conf/test/tools_somatic.config similarity index 100% rename from conf/test/test_tools_somatic.config rename to conf/test/tools_somatic.config diff --git a/conf/test/test_tools_somatic_ascat.config b/conf/test/tools_somatic_ascat.config similarity index 100% rename from conf/test/test_tools_somatic_ascat.config rename to conf/test/tools_somatic_ascat.config diff --git a/conf/test/test_tools_tumoronly.config b/conf/test/tools_tumoronly.config similarity index 100% rename from conf/test/test_tools_tumoronly.config rename to conf/test/tools_tumoronly.config diff --git a/conf/test/test_trimming.config b/conf/test/trimming.config similarity index 100% rename from conf/test/test_trimming.config rename to conf/test/trimming.config diff --git a/conf/test/test_umi.config b/conf/test/umi.config similarity index 100% rename from conf/test/test_umi.config rename to conf/test/umi.config diff --git a/conf/test/test_use_gatk_spark.config b/conf/test/use_gatk_spark.config similarity index 100% rename from conf/test/test_use_gatk_spark.config rename to conf/test/use_gatk_spark.config diff --git a/conf/test/test_variantcalling_channels.config b/conf/test/variantcalling_channels.config similarity index 100% rename from conf/test/test_variantcalling_channels.config rename to conf/test/variantcalling_channels.config diff --git a/conf/test/test_full.config b/conf/test_full.config similarity index 100% rename from conf/test/test_full.config rename to conf/test_full.config diff --git a/conf/test/test_full_germline.config b/conf/test_full_germline.config similarity index 100% rename from conf/test/test_full_germline.config rename to conf/test_full_germline.config diff --git a/nextflow.config b/nextflow.config index 8d0dd50a29..f6fcb16b47 100644 --- a/nextflow.config +++ b/nextflow.config @@ -239,35 +239,36 @@ profiles { includeConfig 'conf/public_aws_ecr.config' } // Basic test profile for CI - test { includeConfig 'conf/test/test.config' } + test { includeConfig 'conf/test.config' } + test_cache { includeConfig 'conf/test/cache.config' } // Extra test profiles for full tests on AWS - test_full { includeConfig 'conf/test/test_full.config' } - test_full_germline { includeConfig 'conf/test/test_full_germline.config' } + test_full { includeConfig 'conf/test_full.config' } + test_full_germline { includeConfig 'conf/test_full_germline.config' } // Extra test profiles for more complete CI - alignment_to_fastq { includeConfig 'conf/test/test_alignment_to_fastq.config' } - annotation { includeConfig 'conf/test/test_annotation.config' } - markduplicates_bam { includeConfig 'conf/test/test_markduplicates_bam.config' } - markduplicates_cram { includeConfig 'conf/test/test_markduplicates_cram.config' } - no_intervals { includeConfig 'conf/test/test_no_intervals.config' } - pair { includeConfig 'conf/test/test_pair.config' } - prepare_recalibration_bam { includeConfig 'conf/test/test_prepare_recalibration_bam.config' } - prepare_recalibration_cram { includeConfig 'conf/test/test_prepare_recalibration_cram.config' } - recalibrate_bam { includeConfig 'conf/test/test_recalibrate_bam.config' } - recalibrate_cram { includeConfig 'conf/test/test_recalibrate_cram.config' } - save_bam_mapped { includeConfig 'conf/test/test_save_bam_mapped.config' } - skip_bqsr { includeConfig 'conf/test/test_skip_bqsr.config' } - skip_markduplicates { includeConfig 'conf/test/test_skip_markduplicates.config' } - split_fastq { includeConfig 'conf/test/test_split_fastq.config' } - targeted { includeConfig 'conf/test/test_targeted.config' } - tools { includeConfig 'conf/test/test_tools.config' } - tools_germline { includeConfig 'conf/test/test_tools_germline.config' } - tools_somatic { includeConfig 'conf/test/test_tools_somatic.config' } - tools_somatic_ascat { includeConfig 'conf/test/test_tools_somatic_ascat.config' } - tools_tumoronly { includeConfig 'conf/test/test_tools_tumoronly.config' } - trimming { includeConfig 'conf/test/test_trimming.config' } - umi { includeConfig 'conf/test/test_umi.config' } - use_gatk_spark { includeConfig 'conf/test/test_use_gatk_spark.config' } - variantcalling_channels { includeConfig 'conf/test/test_variantcalling_channels.config' } + alignment_to_fastq { includeConfig 'conf/test/alignment_to_fastq.config' } + annotation { includeConfig 'conf/test/annotation.config' } + markduplicates_bam { includeConfig 'conf/test/markduplicates_bam.config' } + markduplicates_cram { includeConfig 'conf/test/markduplicates_cram.config' } + no_intervals { includeConfig 'conf/test/no_intervals.config' } + pair { includeConfig 'conf/test/pair.config' } + prepare_recalibration_bam { includeConfig 'conf/test/prepare_recalibration_bam.config' } + prepare_recalibration_cram { includeConfig 'conf/test/prepare_recalibration_cram.config' } + recalibrate_bam { includeConfig 'conf/test/recalibrate_bam.config' } + recalibrate_cram { includeConfig 'conf/test/recalibrate_cram.config' } + save_bam_mapped { includeConfig 'conf/test/save_bam_mapped.config' } + skip_bqsr { includeConfig 'conf/test/skip_bqsr.config' } + skip_markduplicates { includeConfig 'conf/test/skip_markduplicates.config' } + split_fastq { includeConfig 'conf/test/split_fastq.config' } + targeted { includeConfig 'conf/test/targeted.config' } + tools { includeConfig 'conf/test/tools.config' } + tools_germline { includeConfig 'conf/test/tools_germline.config' } + tools_somatic { includeConfig 'conf/test/tools_somatic.config' } + tools_somatic_ascat { includeConfig 'conf/test/tools_somatic_ascat.config' } + tools_tumoronly { includeConfig 'conf/test/tools_tumoronly.config' } + trimming { includeConfig 'conf/test/trimming.config' } + umi { includeConfig 'conf/test/umi.config' } + use_gatk_spark { includeConfig 'conf/test/use_gatk_spark.config' } + variantcalling_channels { includeConfig 'conf/test/variantcalling_channels.config' } } // Load igenomes.config if required diff --git a/tests/test_aligner_bwamem.yml b/tests/test_aligner_bwamem.yml index 883ea6bb5c..fe11f2ad9e 100644 --- a/tests/test_aligner_bwamem.yml +++ b/tests/test_aligner_bwamem.yml @@ -1,5 +1,5 @@ - name: Run bwamem - command: nextflow run main.nf -profile test --aligner bwa-mem --save_reference --outdir results + command: nextflow run main.nf -profile test_cache --aligner bwa-mem --save_reference --outdir results tags: - aligner - bwamem @@ -74,7 +74,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Build only index with bwa - command: nextflow run main.nf -profile test --build_only_index --input=false --outdir results + command: nextflow run main.nf -profile test_cache --build_only_index --input=false --outdir results tags: - aligner - build_only_index diff --git a/tests/test_aligner_bwamem2.yml b/tests/test_aligner_bwamem2.yml index 347f2f3ee4..92a83b66fe 100644 --- a/tests/test_aligner_bwamem2.yml +++ b/tests/test_aligner_bwamem2.yml @@ -1,5 +1,5 @@ - name: Run bwamem2 - command: nextflow run main.nf -profile test --aligner bwa-mem2 --save_reference --outdir results + command: nextflow run main.nf -profile test_cache --aligner bwa-mem2 --save_reference --outdir results tags: - aligner - bwamem2 @@ -74,7 +74,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Build only index with bwa-mem2 - command: nextflow run main.nf -profile test --build_only_index --aligner bwa-mem2 --input=false --outdir results + command: nextflow run main.nf -profile test_cache --build_only_index --aligner bwa-mem2 --input=false --outdir results tags: - aligner - build_only_index diff --git a/tests/test_aligner_dragmap.yml b/tests/test_aligner_dragmap.yml index ce5d7bb715..2b580760a7 100644 --- a/tests/test_aligner_dragmap.yml +++ b/tests/test_aligner_dragmap.yml @@ -1,5 +1,5 @@ - name: Run dragmap - command: nextflow run main.nf -profile test --aligner dragmap --save_reference --outdir results + command: nextflow run main.nf -profile test_cache --aligner dragmap --save_reference --outdir results tags: - aligner - dragmap @@ -87,7 +87,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Build only index with dragmap - command: nextflow run main.nf -profile test --build_only_index --aligner dragmap --input=false --outdir results + command: nextflow run main.nf -profile test_cache --build_only_index --aligner dragmap --input=false --outdir results tags: - aligner - build_only_index diff --git a/tests/test_alignment_to_fastq.yml b/tests/test_alignment_to_fastq.yml index 1b64f422e8..6077b745d4 100644 --- a/tests/test_alignment_to_fastq.yml +++ b/tests/test_alignment_to_fastq.yml @@ -1,5 +1,5 @@ - name: Run alignment to fastq and then remap on bam files - command: nextflow run main.nf -profile test,alignment_to_fastq --outdir results + command: nextflow run main.nf -profile test_cache,alignment_to_fastq --outdir results tags: - alignment_to_fastq - input_bam diff --git a/tests/test_annotation_cache.yml b/tests/test_annotation_cache.yml index 7a83790d43..b8f5179b1a 100644 --- a/tests/test_annotation_cache.yml +++ b/tests/test_annotation_cache.yml @@ -1,5 +1,5 @@ - name: Only download annotation cache - command: nextflow run main.nf -profile test,annotation --tools merge --download_cache --input=false --build_only_index --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools merge --download_cache --input=false --build_only_index --outdir results tags: - annotation - cache @@ -10,7 +10,7 @@ - path: results/annotation should_exist: false - name: Download annotation cache and annotate using snpEff and VEP - command: nextflow run main.nf -profile test,annotation --tools snpeff,vep --download_cache --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools snpeff,vep --download_cache --outdir results tags: - annotation - cache diff --git a/tests/test_annotation_merge.yml b/tests/test_annotation_merge.yml index ccb55ecc6a..0a87db1278 100644 --- a/tests/test_annotation_merge.yml +++ b/tests/test_annotation_merge.yml @@ -1,5 +1,5 @@ - name: Run snpEff followed by VEP - command: nextflow run main.nf -profile test,annotation --tools merge --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools merge --outdir results tags: - annotation - merge @@ -28,7 +28,7 @@ - path: results/reports/EnsemblVEP/test/test_VEP.ann.summary.html should_exist: false - name: Run VEP and snpEff followed by VEP - command: nextflow run main.nf -profile test,annotation --tools merge,snpeff,vep --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools merge,snpeff,vep --outdir results tags: - annotation - merge diff --git a/tests/test_annotation_snpeff.yml b/tests/test_annotation_snpeff.yml index ab21f15b1b..1d573f6da4 100644 --- a/tests/test_annotation_snpeff.yml +++ b/tests/test_annotation_snpeff.yml @@ -1,5 +1,5 @@ - name: Run snpEff - command: nextflow run main.nf -profile test,annotation --tools snpeff --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools snpeff --outdir results tags: - annotation - snpeff diff --git a/tests/test_annotation_vep.yml b/tests/test_annotation_vep.yml index 2e3a138312..275cbdb384 100644 --- a/tests/test_annotation_vep.yml +++ b/tests/test_annotation_vep.yml @@ -1,5 +1,5 @@ - name: Run VEP - command: nextflow run main.nf -profile test,annotation --tools vep --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools vep --outdir results tags: - annotation - vep @@ -12,7 +12,7 @@ - path: results/reports/EnsemblVEP/test/test_VEP.ann.summary.html contains: ["test.vcf.gzOutput filetest_VEP.ann.vcf.gz"] - name: Run VEP with fasta - command: nextflow run main.nf -profile test,annotation --tools vep --vep_include_fasta --outdir results + command: nextflow run main.nf -profile test_cache,annotation --tools vep --vep_include_fasta --outdir results tags: - annotation - vep diff --git a/tests/test_cnvkit.yml b/tests/test_cnvkit.yml index 1bf519aae0..ef959455fc 100644 --- a/tests/test_cnvkit.yml +++ b/tests/test_cnvkit.yml @@ -1,5 +1,5 @@ - name: Run variant calling on somatic samples with cnvkit - command: nextflow run main.nf -profile test,tools_somatic --tools cnvkit --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools cnvkit --outdir results tags: - cnvkit - copy_number_calling @@ -58,7 +58,7 @@ - path: results/cnvkit should_exist: false - name: Run variant calling on tumor_only sample with cnvkit - command: nextflow run main.nf -profile test,tools_tumoronly --tools cnvkit --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools cnvkit --outdir results tags: - cnvkit - copy_number_calling @@ -89,7 +89,7 @@ - path: results/cnvkit should_exist: false - name: Run variant calling on germline sample with cnvkit - command: nextflow run main.nf -profile test,tools_germline --tools cnvkit --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools cnvkit --outdir results tags: - cnvkit - copy_number_calling @@ -122,7 +122,7 @@ - path: results/cnvkit should_exist: false - name: Run variant calling on somatic samples with cnvkit and skip variant calling on matched normal - command: nextflow run main.nf -profile test,tools_somatic --tools cnvkit --only_paired_variant_calling --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools cnvkit --only_paired_variant_calling --outdir results tags: - cnvkit - somatic diff --git a/tests/test_concat_germline_vcfs.yml b/tests/test_concat_germline_vcfs.yml index 97a2453fbc..8a32231ac8 100644 --- a/tests/test_concat_germline_vcfs.yml +++ b/tests/test_concat_germline_vcfs.yml @@ -1,5 +1,5 @@ - name: Run all germline variant callers and check for existence of concatenated vcf-files - command: nextflow run main.nf -profile test --input ./tests/csv/3.0/mapped_joint_bam.csv --concatenate_vcfs --tools deepvariant,freebayes,haplotypecaller,manta,mpileup,strelka,tiddit --step variant_calling --outdir results + command: nextflow run main.nf -profile test_cache --input ./tests/csv/3.0/mapped_joint_bam.csv --concatenate_vcfs --tools deepvariant,freebayes,haplotypecaller,manta,mpileup,strelka,tiddit --step variant_calling --outdir results tags: - concatenate_vcfs files: diff --git a/tests/test_controlfreec.yml b/tests/test_controlfreec.yml index 3806a76ade..9c84ad8fd0 100644 --- a/tests/test_controlfreec.yml +++ b/tests/test_controlfreec.yml @@ -1,5 +1,5 @@ - name: Run variant calling on somatic samples with controlfreec - command: nextflow run main.nf -profile test,tools_somatic --tools controlfreec --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools controlfreec --outdir results tags: - controlfreec - somatic @@ -53,7 +53,7 @@ - path: results/cnvkit should_exist: false - name: Run variant calling on somatic samples with controlfreec without intervals - command: nextflow run main.nf -profile test,tools_somatic --tools controlfreec --no_intervals -stub-run --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools controlfreec --no_intervals -stub-run --outdir results tags: - controlfreec - no_intervals @@ -106,7 +106,7 @@ - path: results/mpileup should_exist: false - name: Run variant calling on tumor_only sample with controlfreec - command: nextflow run main.nf -profile test,tools_tumoronly --tools controlfreec -stub-run --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools controlfreec -stub-run --outdir results tags: - controlfreec - tumor_only diff --git a/tests/test_deepvariant.yml b/tests/test_deepvariant.yml index d73b390296..ce378e1e76 100644 --- a/tests/test_deepvariant.yml +++ b/tests/test_deepvariant.yml @@ -1,5 +1,5 @@ - name: Run variant calling on germline sample with deepvariant - command: nextflow run main.nf -profile test,tools_germline --tools deepvariant --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools deepvariant --outdir results tags: - deepvariant - germline @@ -25,7 +25,7 @@ - path: results/deepvariant should_exist: false - name: Run variant calling on germline sample with deepvariant without intervals - command: nextflow run main.nf -profile test,tools_germline --tools deepvariant --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools deepvariant --no_intervals --outdir results tags: - deepvariant - germline diff --git a/tests/test_fastp.yml b/tests/test_fastp.yml index deefabb552..5e9ac2dc10 100644 --- a/tests/test_fastp.yml +++ b/tests/test_fastp.yml @@ -1,5 +1,5 @@ - name: Run trimming pipeline - command: nextflow run main.nf -profile test,trimming --save_trimmed --outdir results + command: nextflow run main.nf -profile test_cache,trimming --save_trimmed --outdir results tags: - fastp - preprocessing @@ -55,7 +55,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Run split fastq module - command: nextflow run main.nf -profile test,split_fastq --outdir results + command: nextflow run main.nf -profile test_cache,split_fastq --outdir results tags: - fastp - preprocessing diff --git a/tests/test_freebayes.yml b/tests/test_freebayes.yml index 2bb64eb531..0808b7ee7d 100644 --- a/tests/test_freebayes.yml +++ b/tests/test_freebayes.yml @@ -1,5 +1,5 @@ - name: Run variant calling on germline sample with freebayes - command: nextflow run main.nf -profile test,targeted --tools freebayes --outdir results + command: nextflow run main.nf -profile test_cache,targeted --tools freebayes --outdir results tags: - freebayes - germline @@ -74,7 +74,7 @@ - path: results/freebayes should_exist: false - name: Run variant calling on germline sample with freebayes without intervals - command: nextflow run main.nf -profile test --tools freebayes --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache --tools freebayes --no_intervals --outdir results tags: - freebayes - germline @@ -148,7 +148,7 @@ - path: results/freebayes should_exist: false - name: Run variant calling on somatic sample with freebayes - command: nextflow run main.nf -profile test,pair,targeted --tools freebayes --outdir results + command: nextflow run main.nf -profile test_cache,pair,targeted --tools freebayes --outdir results tags: - freebayes - somatic @@ -279,7 +279,7 @@ - path: results/freebayes should_exist: false - name: Run variant calling on somatic sample with freebayes without intervals - command: nextflow run main.nf -profile test,pair,targeted --tools freebayes --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,pair,targeted --tools freebayes --no_intervals --outdir results tags: - freebayes - somatic @@ -393,7 +393,7 @@ - path: results/freebayes should_exist: false - name: Run variant calling on tumor_only sample with freebayes - command: nextflow run main.nf -profile test,tools_tumoronly --tools freebayes --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools freebayes --outdir results tags: - freebayes - tumor_only @@ -417,7 +417,7 @@ - path: results/freebayes should_exist: false - name: Run variant calling on tumor_only sample with freebayes without intervals - command: nextflow run main.nf -profile test,tools_tumoronly --tools freebayes --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools freebayes --no_intervals --outdir results tags: - freebayes - no_intervals diff --git a/tests/test_gatk4_spark.yml b/tests/test_gatk4_spark.yml index 4001b715ae..e7632b3d92 100644 --- a/tests/test_gatk4_spark.yml +++ b/tests/test_gatk4_spark.yml @@ -1,5 +1,5 @@ - name: Run default pipeline with gatk4_spark - command: nextflow run main.nf -profile test,use_gatk_spark --outdir results + command: nextflow run main.nf -profile test_cache,use_gatk_spark --outdir results tags: - gatk4_spark - preprocessing @@ -49,7 +49,7 @@ - path: results/preprocessing/mapped/ should_exist: false - name: Run default pipeline with gatk4_spark & skipping all QC steps - command: nextflow run main.nf -profile test,use_gatk_spark --skip_tools fastqc,markduplicates_report,mosdepth,multiqc,samtools --outdir results + command: nextflow run main.nf -profile test_cache,use_gatk_spark --skip_tools fastqc,markduplicates_report,mosdepth,multiqc,samtools --outdir results tags: - gatk4_spark - preprocessing @@ -82,7 +82,7 @@ - path: results/reports/samtools should_exist: false - name: Run save_output_as_bam with gatk4 spark - command: nextflow run main.nf -profile test,use_gatk_spark --save_output_as_bam --outdir results + command: nextflow run main.nf -profile test_cache,use_gatk_spark --save_output_as_bam --outdir results tags: - gatk4_spark - preprocessing diff --git a/tests/test_haplotypecaller.yml b/tests/test_haplotypecaller.yml index 8e105fac13..4ec225367b 100644 --- a/tests/test_haplotypecaller.yml +++ b/tests/test_haplotypecaller.yml @@ -1,5 +1,5 @@ - name: Run variant calling on germline sample with haplotypecaller - command: nextflow run main.nf -profile test,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --outdir results + command: nextflow run main.nf -profile test_cache,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --outdir results tags: - germline - haplotypecaller @@ -35,7 +35,7 @@ - path: results/haplotypecaller should_exist: false - name: Run variant calling on germline sample with haplotypecaller without intervals - command: nextflow run main.nf -profile test,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --no_intervals --outdir results tags: - germline - haplotypecaller diff --git a/tests/test_haplotypecaller_skip_filter.yml b/tests/test_haplotypecaller_skip_filter.yml index 085c3aaee3..84f6362021 100644 --- a/tests/test_haplotypecaller_skip_filter.yml +++ b/tests/test_haplotypecaller_skip_filter.yml @@ -1,5 +1,5 @@ - name: Run variant calling on germline sample with haplotypecaller and skip filter - command: nextflow run main.nf -profile test,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --skip_tools haplotypecaller_filter --outdir results + command: nextflow run main.nf -profile test_cache,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --skip_tools haplotypecaller_filter --outdir results tags: - germline - haplotypecaller_skip_filter @@ -35,7 +35,7 @@ - path: results/haplotypecaller should_exist: false - name: Run variant calling on germline sample with haplotypecaller without intervals and skip filter - command: nextflow run main.nf -profile test,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --skip_tools haplotypecaller_filter --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,targeted --input ./tests/csv/3.0/mapped_single_bam.csv --tools haplotypecaller --step variant_calling --skip_tools haplotypecaller_filter --no_intervals --outdir results tags: - germline - haplotypecaller_skip_filter diff --git a/tests/test_intervals.yml b/tests/test_intervals.yml index ba0b887cf7..ed12b1aca4 100644 --- a/tests/test_intervals.yml +++ b/tests/test_intervals.yml @@ -1,5 +1,5 @@ - name: Run default pipeline with target bed - command: nextflow run main.nf -profile test,targeted --outdir results + command: nextflow run main.nf -profile test_cache,targeted --outdir results tags: - intervals - preprocessing @@ -58,7 +58,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Run pipeline with intervals false - command: nextflow run main.nf -profile test --intervals false --save_reference --outdir results + command: nextflow run main.nf -profile test_cache --intervals false --save_reference --outdir results tags: - intervals - intervals_false @@ -109,7 +109,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Run default pipeline without intervals - command: nextflow run main.nf -profile test,no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,no_intervals --outdir results tags: - intervals - no_intervals diff --git a/tests/test_joint_germline.yml b/tests/test_joint_germline.yml index 2b6821f9fe..2466f440c8 100644 --- a/tests/test_joint_germline.yml +++ b/tests/test_joint_germline.yml @@ -1,5 +1,5 @@ - name: Run joint germline variant calling with haplotypecaller - command: nextflow run main.nf -profile test,targeted --input ./tests/csv/3.0/mapped_joint_bam.csv --tools haplotypecaller --step variant_calling --joint_germline --outdir results + command: nextflow run main.nf -profile test_cache,targeted --input ./tests/csv/3.0/mapped_joint_bam.csv --tools haplotypecaller --step variant_calling --joint_germline --outdir results tags: - germline - joint_germline diff --git a/tests/test_manta.yml b/tests/test_manta.yml index 635ccef01e..1e53587f6e 100644 --- a/tests/test_manta.yml +++ b/tests/test_manta.yml @@ -1,5 +1,5 @@ - name: Run variant calling on germline sample with manta - command: nextflow run main.nf -profile test,tools_germline --tools manta --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools manta --outdir results tags: - germline - manta @@ -23,7 +23,7 @@ - path: results/manta should_exist: false - name: Run variant calling on germline sample with manta without intervals - command: nextflow run main.nf -profile test,tools_germline --tools manta --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools manta --no_intervals --outdir results tags: - germline - manta @@ -54,7 +54,7 @@ - path: results/manta should_exist: false - name: Run variant calling on tumor_only sample with manta - command: nextflow run main.nf -profile test,tools_tumoronly --tools manta --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools manta --outdir results tags: - manta - tumor_only @@ -78,7 +78,7 @@ - path: results/manta should_exist: false - name: Run variant calling on tumor_only sample with manta without intervals - command: nextflow run main.nf -profile test,tools_tumoronly --tools manta --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools manta --no_intervals --outdir results tags: - manta - no_intervals @@ -109,7 +109,7 @@ - path: results/manta should_exist: false - name: Run variant calling on somatic sample with manta - command: nextflow run main.nf -profile test,tools_somatic --tools manta --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools manta --outdir results tags: - manta - somatic @@ -157,7 +157,7 @@ - path: results/manta should_exist: false - name: Run variant calling on somatic sample with manta without intervals - command: nextflow run main.nf -profile test,tools_somatic --tools manta --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools manta --no_intervals --outdir results tags: - manta - no_intervals diff --git a/tests/test_markduplicates_from_bam.yml b/tests/test_markduplicates_from_bam.yml index 4415d1c389..dee006127f 100644 --- a/tests/test_markduplicates_from_bam.yml +++ b/tests/test_markduplicates_from_bam.yml @@ -1,5 +1,5 @@ - name: Run markduplicates starting from BAM - command: nextflow run main.nf -profile test,markduplicates_bam --outdir results + command: nextflow run main.nf -profile test_cache,markduplicates_bam --outdir results tags: - input_bam - markduplicates @@ -49,7 +49,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # conda changes md5sums for test - name: Run skip markduplicates bam from step markduplicates - command: nextflow run main.nf -profile test,markduplicates_bam,skip_markduplicates --outdir results + command: nextflow run main.nf -profile test_cache,markduplicates_bam,skip_markduplicates --outdir results tags: - input_bam - markduplicates diff --git a/tests/test_markduplicates_from_cram.yml b/tests/test_markduplicates_from_cram.yml index 69816f44ba..ab2e020c85 100644 --- a/tests/test_markduplicates_from_cram.yml +++ b/tests/test_markduplicates_from_cram.yml @@ -1,5 +1,5 @@ - name: Run markduplicates starting from CRAM - command: nextflow run main.nf -profile test,markduplicates_cram --outdir results + command: nextflow run main.nf -profile test_cache,markduplicates_cram --outdir results tags: - input_cram - markduplicates @@ -51,7 +51,7 @@ - path: results/preprocessing/mapped/ should_exist: false - name: Run skip markduplicates cram from step markduplicates - command: nextflow run main.nf -profile test,markduplicates_cram,skip_markduplicates --outdir results + command: nextflow run main.nf -profile test_cache,markduplicates_cram,skip_markduplicates --outdir results tags: - input_cram - markduplicates diff --git a/tests/test_mpileup.yml b/tests/test_mpileup.yml index 472c83f0d3..6ac744f434 100644 --- a/tests/test_mpileup.yml +++ b/tests/test_mpileup.yml @@ -1,5 +1,5 @@ - name: Run variant calling on tumor_only sample to test mpileup - command: nextflow run main.nf -profile test,tools_tumoronly --tools mpileup --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools mpileup --outdir results tags: - tumor_only - mpileup @@ -10,7 +10,7 @@ - path: results/mpileup should_exist: false - name: Run variant calling on tumor_only sample to test mpileup without intervals - command: nextflow run main.nf -profile test,tools_tumoronly --tools mpileup --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools mpileup --no_intervals --outdir results tags: - tumor_only - mpileup @@ -28,7 +28,7 @@ - path: results/mpileup should_exist: false - name: Run variant calling on germline sample to test mpileup - command: nextflow run main.nf -profile test,tools_germline --tools mpileup --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools mpileup --outdir results tags: - germline - mpileup @@ -39,7 +39,7 @@ - path: results/mpileup should_exist: false - name: Run variant calling on germline sample to test mpileup without intervals - command: nextflow run main.nf -profile test,tools_germline --tools mpileup --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools mpileup --no_intervals --outdir results tags: - germline - mpileup diff --git a/tests/test_msisensorpro.yml b/tests/test_msisensorpro.yml index 0443f47be8..912a33adfc 100644 --- a/tests/test_msisensorpro.yml +++ b/tests/test_msisensorpro.yml @@ -1,5 +1,5 @@ - name: Run variant calling on somatic sample with msisensor-pro - command: nextflow run main.nf -profile test,tools_somatic --tools msisensorpro --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools msisensorpro --outdir results tags: - msisensorpro - somatic @@ -13,7 +13,7 @@ - path: results/msisensorpro should_exist: false - name: Build only index with msisensorpro - command: nextflow run main.nf -profile test --build_only_index --tools msisensorpro --input=false --outdir results + command: nextflow run main.nf -profile test_cache --build_only_index --tools msisensorpro --input=false --outdir results tags: - build_only_index - msisensorpro diff --git a/tests/test_mutect2.yml b/tests/test_mutect2.yml index ac83aac38f..0329e827d3 100644 --- a/tests/test_mutect2.yml +++ b/tests/test_mutect2.yml @@ -1,5 +1,5 @@ - name: Run variant calling on tumor only sample with mutect2 - command: nextflow run main.nf -profile test,tools_tumoronly --tools mutect2 --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools mutect2 --outdir results tags: - mutect2 - tumor_only @@ -39,7 +39,7 @@ - path: results/mutect2 should_exist: false - name: Run variant calling on tumor only sample with mutect2 without intervals - command: nextflow run main.nf -profile test,tools_tumoronly --tools mutect2 --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools mutect2 --no_intervals --outdir results tags: - mutect2 - no_intervals diff --git a/tests/test_prepare_recalibration_from_bam.yml b/tests/test_prepare_recalibration_from_bam.yml index b5ab788e78..3d2560058c 100644 --- a/tests/test_prepare_recalibration_from_bam.yml +++ b/tests/test_prepare_recalibration_from_bam.yml @@ -1,5 +1,5 @@ - name: Run prepare_recalibration starting from bam - command: nextflow run main.nf -profile test,prepare_recalibration_bam --outdir results + command: nextflow run main.nf -profile test_cache,prepare_recalibration_bam --outdir results tags: - input_bam - prepare_recalibration @@ -37,7 +37,7 @@ - path: results/preprocessing/markduplicates/ should_exist: false - name: Run prepare_recalibration starting from bam and skip baserecalibration - command: nextflow run main.nf -profile test,prepare_recalibration_bam,skip_bqsr --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,prepare_recalibration_bam,skip_bqsr --tools strelka --outdir results tags: - input_bam - prepare_recalibration diff --git a/tests/test_prepare_recalibration_from_cram.yml b/tests/test_prepare_recalibration_from_cram.yml index d207c6f190..d69fa91525 100644 --- a/tests/test_prepare_recalibration_from_cram.yml +++ b/tests/test_prepare_recalibration_from_cram.yml @@ -1,5 +1,5 @@ - name: Run prepare_recalibration starting from cram - command: nextflow run main.nf -profile test,prepare_recalibration_cram --outdir results + command: nextflow run main.nf -profile test_cache,prepare_recalibration_cram --outdir results tags: - input_cram - prepare_recalibration @@ -31,7 +31,7 @@ - path: results/preprocessing/markduplicates/ should_exist: false - name: Run prepare_recalibration starting from cram and skip baserecalibration - command: nextflow run main.nf -profile test,prepare_recalibration_cram,skip_bqsr --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,prepare_recalibration_cram,skip_bqsr --tools strelka --outdir results tags: - input_cram - prepare_recalibration diff --git a/tests/test_recalibrate_from_bam.yml b/tests/test_recalibrate_from_bam.yml index 41d0c91e4a..cc1a3e114a 100644 --- a/tests/test_recalibrate_from_bam.yml +++ b/tests/test_recalibrate_from_bam.yml @@ -1,5 +1,5 @@ - name: Run Recalibration starting from bam - command: nextflow run main.nf -profile test,recalibrate_bam --outdir results + command: nextflow run main.nf -profile test_cache,recalibrate_bam --outdir results tags: - input_bam - recalibrate @@ -33,7 +33,7 @@ - path: results/preprocessing/markduplicates/ should_exist: false - name: Run Recalibration starting from bam and skip baserecalibration - command: nextflow run main.nf -profile test,recalibrate_bam,skip_bqsr --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,recalibrate_bam,skip_bqsr --tools strelka --outdir results tags: - input_bam - recalibrate diff --git a/tests/test_recalibrate_from_cram.yml b/tests/test_recalibrate_from_cram.yml index 5f5d0badd3..e843424f50 100644 --- a/tests/test_recalibrate_from_cram.yml +++ b/tests/test_recalibrate_from_cram.yml @@ -1,5 +1,5 @@ - name: Run Recalibration starting from cram - command: nextflow run main.nf -profile test,recalibrate_cram --outdir results + command: nextflow run main.nf -profile test_cache,recalibrate_cram --outdir results tags: - input_cram - recalibrate @@ -29,7 +29,7 @@ - path: results/preprocessing/markduplicates/ should_exist: false - name: Run Recalibration starting from cram and skip baserecalibration - command: nextflow run main.nf -profile test,recalibrate_cram,skip_bqsr --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,recalibrate_cram,skip_bqsr --tools strelka --outdir results tags: - input_cram - recalibrate diff --git a/tests/test_samplesheet_validation_spaces.yml b/tests/test_samplesheet_validation_spaces.yml index 025d0815c0..4ae615d441 100644 --- a/tests/test_samplesheet_validation_spaces.yml +++ b/tests/test_samplesheet_validation_spaces.yml @@ -1,5 +1,5 @@ - name: Test that pipeline fail when sample/patient name contain space - command: nextflow run main.nf -profile test --input ./tests/csv/3.0/sample_with_space.csv --outdir results + command: nextflow run main.nf -profile test_cache --input ./tests/csv/3.0/sample_with_space.csv --outdir results tags: - sample_with_space - validation_checks diff --git a/tests/test_save_mapped.yml b/tests/test_save_mapped.yml index ef21fcb3fb..8b06af1ad7 100644 --- a/tests/test_save_mapped.yml +++ b/tests/test_save_mapped.yml @@ -1,5 +1,5 @@ - name: Run save_mapped - command: nextflow run main.nf -profile test --save_mapped --outdir results + command: nextflow run main.nf -profile test_cache --save_mapped --outdir results tags: - default_extended - preprocessing diff --git a/tests/test_save_output_as_bam_only.yml b/tests/test_save_output_as_bam_only.yml index 0b38e3bdcb..7b8c3f5f45 100644 --- a/tests/test_save_output_as_bam_only.yml +++ b/tests/test_save_output_as_bam_only.yml @@ -1,5 +1,5 @@ - name: Run save_output_as_bam - command: nextflow run main.nf -profile test --save_output_as_bam --outdir results + command: nextflow run main.nf -profile test_cache --save_output_as_bam --outdir results tags: - default_extended - preprocessing diff --git a/tests/test_skip_all_qc.yml b/tests/test_skip_all_qc.yml index 54b4c2069f..aa8a62ba4c 100644 --- a/tests/test_skip_all_qc.yml +++ b/tests/test_skip_all_qc.yml @@ -1,5 +1,5 @@ - name: Run default pipeline with skipping all QC steps - command: nextflow run main.nf -profile test --skip_tools fastqc,markduplicates_report,mosdepth,multiqc,samtools --outdir results + command: nextflow run main.nf -profile test_cache --skip_tools fastqc,markduplicates_report,mosdepth,multiqc,samtools --outdir results tags: - default_extended - preprocessing diff --git a/tests/test_skip_markduplicates.yml b/tests/test_skip_markduplicates.yml index 9e7fda0712..9c20eb3c89 100644 --- a/tests/test_skip_markduplicates.yml +++ b/tests/test_skip_markduplicates.yml @@ -1,5 +1,5 @@ - name: Run default pipeline with skipping Markduplicates - command: nextflow run main.nf -profile test,skip_markduplicates --outdir results + command: nextflow run main.nf -profile test_cache,skip_markduplicates --outdir results tags: - default_extended - preprocessing @@ -54,7 +54,7 @@ - path: results/preprocessing/mapped/test/test.sorted.bam should_exist: false - name: Run default pipeline with skipping Markduplicates with save_mapped - command: nextflow run main.nf -profile test,skip_markduplicates --save_mapped --outdir results + command: nextflow run main.nf -profile test_cache,skip_markduplicates --save_mapped --outdir results tags: - default_extended - preprocessing @@ -108,7 +108,7 @@ - path: results/preprocessing/mapped/test/test.bam should_exist: false - name: Run default pipeline with skipping Markduplicates with save_mapped & save_output_as_bam - command: nextflow run main.nf -profile test,skip_markduplicates --save_mapped --save_output_as_bam --outdir results + command: nextflow run main.nf -profile test_cache,skip_markduplicates --save_mapped --save_output_as_bam --outdir results tags: - default_extended - preprocessing diff --git a/tests/test_strelka.yml b/tests/test_strelka.yml index 94aff2f155..2a3a24c936 100644 --- a/tests/test_strelka.yml +++ b/tests/test_strelka.yml @@ -1,5 +1,5 @@ - name: Skip variant calling on matched normal - command: nextflow run main.nf -profile test,variantcalling_channels --tools strelka --only_paired_variant_calling --outdir results + command: nextflow run main.nf -profile test_cache,variantcalling_channels --tools strelka --only_paired_variant_calling --outdir results tags: - somatic - strelka @@ -77,7 +77,7 @@ - path: results/variant_calling/strelka/sample3/sample3.strelka.genome.vcf.gz.tbi should_exist: false - name: Run variant calling on germline sample with strelka - command: nextflow run main.nf -profile test,tools_germline --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools strelka --outdir results tags: - germline - strelka @@ -105,7 +105,7 @@ - path: results/strelka should_exist: false - name: Run variant calling on germline sample with strelka without intervals - command: nextflow run main.nf -profile test,tools_germline --tools strelka --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools strelka --no_intervals --outdir results tags: - germline - strelka @@ -140,7 +140,7 @@ - path: results/strelka should_exist: false - name: Run variant calling on tumor only sample with strelka - command: nextflow run main.nf -profile test,tools_tumoronly --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools strelka --outdir results tags: - strelka - tumor_only @@ -168,7 +168,7 @@ - path: results/strelka should_exist: false - name: Run variant calling on tumor only sample with strelka without intervals - command: nextflow run main.nf -profile test,tools_tumoronly --tools strelka --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools strelka --no_intervals --outdir results tags: - no_intervals - strelka @@ -203,7 +203,7 @@ - path: results/strelka should_exist: false - name: Run variant calling on somatic sample with strelka - command: nextflow run main.nf -profile test,tools_somatic --tools strelka --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools strelka --outdir results tags: - somatic - strelka @@ -255,7 +255,7 @@ - path: results/strelka should_exist: false - name: Run variant calling on somatic sample with strelka without intervals - command: nextflow run main.nf -profile test,tools_somatic --tools strelka --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools strelka --no_intervals --outdir results tags: - no_intervals - somatic diff --git a/tests/test_strelka_bp.yml b/tests/test_strelka_bp.yml index 66c502267f..b89d09550d 100644 --- a/tests/test_strelka_bp.yml +++ b/tests/test_strelka_bp.yml @@ -1,5 +1,5 @@ - name: Run variant calling on somatic sample with Strelka BP - command: nextflow run main.nf -profile test,tools_somatic --tools strelka,manta --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools strelka,manta --outdir results tags: - somatic - strelka_bp @@ -89,7 +89,7 @@ - path: results/strelka should_exist: false - name: Run variant calling on somatic sample with Strelka BP without intervals - command: nextflow run main.nf -profile test,tools_somatic --tools strelka,manta --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools strelka,manta --no_intervals --outdir results tags: - no_intervals - somatic diff --git a/tests/test_tiddit.yml b/tests/test_tiddit.yml index ce9f985359..ad82fb4605 100644 --- a/tests/test_tiddit.yml +++ b/tests/test_tiddit.yml @@ -1,5 +1,5 @@ - name: Run variant calling on somatic sample with tiddit - command: nextflow run main.nf -profile test,tools_somatic --tools tiddit --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools tiddit --outdir results tags: - tiddit - somatic @@ -45,7 +45,7 @@ - path: results/tiddit should_exist: false - name: Run variant calling on germline sample with tiddit - command: nextflow run main.nf -profile test,tools_germline --tools tiddit --outdir results + command: nextflow run main.nf -profile test_cache,tools_germline --tools tiddit --outdir results tags: - tiddit - germline @@ -71,7 +71,7 @@ - path: results/tiddit should_exist: false - name: Run variant calling on tumor_only sample with tiddit - command: nextflow run main.nf -profile test,tools_tumoronly --tools tiddit --outdir results + command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools tiddit --outdir results tags: - tiddit - tumor_only diff --git a/tests/test_tools_manually.yml b/tests/test_tools_manually.yml index 955cd0836a..520d7afe11 100644 --- a/tests/test_tools_manually.yml +++ b/tests/test_tools_manually.yml @@ -1,5 +1,5 @@ - name: Run variant calling on somatic samples with ascat - command: nextflow run main.nf -profile test,tools_somatic_ascat --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic_ascat --outdir results tags: - ascat_manual - manual @@ -35,7 +35,7 @@ - path: results/variant_calling/ascat/sample4_vs_sample3/sample4_vs_sample3.tumour_normalLogR.txt md5sum: 05418a7d814db11808172a4f57d040a1 - name: Run variant calling on somatic sample with mutect2 without intervals - command: nextflow run main.nf -profile test,tools_somatic --tools mutect2 --no_intervals --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools mutect2 --no_intervals --outdir results tags: - mutect2_manual - manual @@ -83,7 +83,7 @@ - path: results/variant_calling/mutect2/sample4_vs_sample3/sample4_vs_sample3.mutect2.vcf.gz.tbi # binary changes md5sums on reruns - name: Run variant calling on somatic sample with mutect2 - command: nextflow run main.nf -profile test,tools_somatic --tools mutect2 --outdir results + command: nextflow run main.nf -profile test_cache,tools_somatic --tools mutect2 --outdir results tags: - mutect2_manual - manual diff --git a/tests/test_tumor_normal_pair.yml b/tests/test_tumor_normal_pair.yml index fe1fd5de21..b616fc2b83 100644 --- a/tests/test_tumor_normal_pair.yml +++ b/tests/test_tumor_normal_pair.yml @@ -1,5 +1,5 @@ - name: Run default pipeline for tumor normal pair - command: nextflow run main.nf -profile test,pair --outdir results + command: nextflow run main.nf -profile test_cache,pair --outdir results tags: - default_extended - preprocessing diff --git a/tests/test_umi.yml b/tests/test_umi.yml index 3cbf6c2c86..19aeac27d3 100644 --- a/tests/test_umi.yml +++ b/tests/test_umi.yml @@ -1,5 +1,5 @@ - name: Run UMI test - command: nextflow run main.nf -profile test,umi --outdir results + command: nextflow run main.nf -profile test_cache,umi --outdir results tags: - preprocessing - umi @@ -51,7 +51,7 @@ - path: results/reports/samtools/test/test.recal.cram.stats # text-based file changes md5sums on reruns # - name: Run UMI TSO test -# command: nextflow run main.nf -profile test,umi_tso --outdir results +# command: nextflow run main.nf -profile test_cache,umi_tso --outdir results # tags: # - umi_tso # - umi From ab8ced47027ddfcffaa515583a0c378679ba4bee Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 12:38:30 +0200 Subject: [PATCH 05/47] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e1412a8c..309ec7558c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#1087](https://github.com/nf-core/sarek/pull/1087) - Back to dev - [#1087](https://github.com/nf-core/sarek/pull/1087) - Minor modules update +- [#1088](https://github.com/nf-core/sarek/pull/1088) - Replace profile `test` by `test_cache` and add a `test` profile with non hidden files ### Fixed From 162ac72ce7bf03c1d9565ee6dffb1658d847cdac Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 12:40:56 +0200 Subject: [PATCH 06/47] test only on docker --- .github/workflows/ci.yml | 2 +- .github/workflows/pytest-workflow.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 743876c1eb..42880df7f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - "latest-everything" test: - "default" - profile: ["docker", "singularity"] + profile: ["docker"] # profile: ["docker", "singularity", "conda"] env: NXF_ANSI_LOG: false diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 394802985f..f9ff393393 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -32,7 +32,7 @@ jobs: fail-fast: false matrix: tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] - profile: ["docker", "singularity"] + profile: ["docker"] # profile: ["docker", "singularity", "conda"] TEST_DATA_BASE: - "test-datasets/data" From 1f40c3dd58f4e87ecb89749dc61b9f234c1d1f5d Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 12:45:40 +0200 Subject: [PATCH 07/47] test with singularity and conda when PR on master --- .github/workflows/pytest-workflow.yml | 2 +- .github/workflows/pytest-workflow_release.yml | 171 ++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pytest-workflow_release.yml diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index f9ff393393..41b7f13784 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -2,7 +2,7 @@ name: pytest-workflow # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: pull_request: - branches: [dev, master] + branches: [dev] # Cancel if a newer run is started concurrency: diff --git a/.github/workflows/pytest-workflow_release.yml b/.github/workflows/pytest-workflow_release.yml new file mode 100644 index 0000000000..3f434c203a --- /dev/null +++ b/.github/workflows/pytest-workflow_release.yml @@ -0,0 +1,171 @@ +name: pytest-workflow +# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +on: + push: + branches: [master] + pull_request: + release: + types: [published] + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + changes: + name: Check for changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'tags' output variable + tags: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/tags.yml" + + test: + name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] + profile: ["docker", "singularity", "conda"] + TEST_DATA_BASE: + - "test-datasets/data" + NXF_VER: + - "23.04.0" + - "latest-everything" + exclude: + - profile: "conda" + tags: concatenate_vcfs + - profile: "conda" + tags: deepvariant + - profile: "conda" + tags: haplotypecaller + - profile: "conda" + tags: merge + - profile: "conda" + tags: snpeff + - profile: "conda" + tags: umi + - profile: "conda" + tags: validation_checks + - profile: "conda" + tags: vep + - profile: "singularity" + tags: concatenate_vcfs + - profile: "singularity" + tags: merge + - profile: "singularity" + tags: validation_checks + env: + NXF_ANSI_LOG: false + TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" + steps: + - name: Check out pipeline code + uses: actions/checkout@v3 + + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo sarek3_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + + - name: Cache test data + id: cache-testdata + uses: actions/cache@v3 + with: + path: test-datasets/ + key: ${{ steps.hash_workspace.outputs.digest }} + + - name: Check out test data + if: steps.cache-testdata.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: nf-core/test-datasets + ref: sarek3 + path: test-datasets/ + + - name: Replace remote paths in samplesheets + run: | + for f in tests/csv/3.0/*csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/modules/=${{ github.workspace }}/test-datasets/=g" $f + echo "========== $f ============" + cat $f + echo "========================================" + done; + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow ${{ matrix.NXF_VER }} + uses: nf-core/setup-nextflow@v1.2.0 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda clean + if: matrix.profile == 'conda' + run: conda clean -a + + - name: Run pytest-workflow + uses: Wandalen/wretry.action@v1.0.11 + with: + command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + attempt_limit: 3 + + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }} + path: | + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity From 51a3eded890408e434179d45c9608727bf1d4a79 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 9 Jun 2023 13:37:38 +0200 Subject: [PATCH 08/47] Apply suggestions from code review Co-authored-by: Anders Sune Pedersen <37172585+asp8200@users.noreply.github.com> --- .github/workflows/pytest-workflow_release.yml | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest-workflow_release.yml b/.github/workflows/pytest-workflow_release.yml index 3f434c203a..6cff0411ac 100644 --- a/.github/workflows/pytest-workflow_release.yml +++ b/.github/workflows/pytest-workflow_release.yml @@ -1,4 +1,4 @@ -name: pytest-workflow +name: pytest-workflow-release # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: push: diff --git a/CHANGELOG.md b/CHANGELOG.md index 309ec7558c..918f492ddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#1087](https://github.com/nf-core/sarek/pull/1087) - Back to dev - [#1087](https://github.com/nf-core/sarek/pull/1087) - Minor modules update -- [#1088](https://github.com/nf-core/sarek/pull/1088) - Replace profile `test` by `test_cache` and add a `test` profile with non hidden files +- [#1088](https://github.com/nf-core/sarek/pull/1088) - Replace profile `test` by `test_cache` and add a `test` profile without hidden files ### Fixed From c19188eaf321b600aa12e428f14c75fd79a7cd1a Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 9 Jun 2023 13:41:04 +0200 Subject: [PATCH 09/47] Update .github/workflows/pytest-workflow_release.yml --- .github/workflows/pytest-workflow_release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest-workflow_release.yml b/.github/workflows/pytest-workflow_release.yml index 6cff0411ac..a2abde262b 100644 --- a/.github/workflows/pytest-workflow_release.yml +++ b/.github/workflows/pytest-workflow_release.yml @@ -3,7 +3,6 @@ name: pytest-workflow-release on: push: branches: [master] - pull_request: release: types: [published] From 4d0a5f300e5759239f79291a7399148061714731 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 13:50:41 +0200 Subject: [PATCH 10/47] update usage --- docs/usage.md | 156 +++++++++++++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 72 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 04ad4f465c..541219ec40 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -433,96 +433,108 @@ For a thorough list, please refer the [Azure Sizes for virtual machines in Azure ## How to test the pipeline -When using default parameters only, sarek runs preprocessing and `Strelka2`. This is reflected in the default test profile: +When using default parameters only, sarek runs preprocessing and `Strelka2`. +This is reflected in the default test profile: ```bash -nextflow run nf-core/sarek -r 3.0.1 -profile test, +nextflow run nf-core/sarek -r 3.2.1 -profile test, --outdir results ``` Expected run output: ```bash -[42/360944] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:BWAMEM1_INDEX (genome.fasta) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:BWAMEM2_INDEX - -[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:DRAGMAP_HASHTABLE - -[2b/92dc0c] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:GATK4_CREATESEQUENCEDICTIONARY (genome.fasta) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:MSISENSORPRO_SCAN - -[58/ecea17] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:SAMTOOLS_FAIDX (genome.fasta) [100%] 1 of 1 ✔ -[9c/121939] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_DBSNP (dbsnp_146.hg38.vcf) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_GERMLINE_RESOURCE - -[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_KNOWN_SNPS - -[28/6f5c14] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_KNOWN_INDELS (mills_and_1000G.indels.vcf) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_PON - -[55/4466e1] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:CREATE_INTERVALS_BED (genome.interval_list) [100%] 1 of 1 ✔ -[f8/df9607] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:GATK4_INTERVALLISTTOBED (genome) [100%] 1 of 1 ✔ -[ff/26467d] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:TABIX_BGZIPTABIX_INTERVAL_SPLIT (chr22_1-40001) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:PREPARE_CNVKIT_REFERENCE:CNVKIT_ANTITARGET - -[- ] process > NFCORE_SAREK:SAREK:PREPARE_CNVKIT_REFERENCE:CNVKIT_REFERENCE - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:SAMTOOLS_VIEW_MAP_MAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:SAMTOOLS_VIEW_UNMAP_UNMAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:SAMTOOLS_VIEW_UNMAP_MAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:SAMTOOLS_VIEW_MAP_UNMAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:SAMTOOLS_MERGE_UNMAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:COLLATE_FASTQ_UNMAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:COLLATE_FASTQ_MAP - -[- ] process > NFCORE_SAREK:SAREK:ALIGNMENT_TO_FASTQ_INPUT:CAT_FASTQ - -[74/cd6685] process > NFCORE_SAREK:SAREK:RUN_FASTQC:FASTQC (test-test_L1) [100%] 1 of 1 ✔ -[bc/ea89a8] process > NFCORE_SAREK:SAREK:GATK4_MAPPING:BWAMEM1_MEM (test) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:GATK4_MAPPING:BWAMEM2_MEM - -[- ] process > NFCORE_SAREK:SAREK:GATK4_MAPPING:DRAGMAP_ALIGN - -[46/35a640] process > NFCORE_SAREK:SAREK:MARKDUPLICATES:GATK4_MARKDUPLICATES (test) [100%] 1 of 1 ✔ -[9a/76cef7] process > NFCORE_SAREK:SAREK:MARKDUPLICATES:INDEX_MARKDUPLICATES (test) [100%] 1 of 1 ✔ -[46/9fe93a] process > NFCORE_SAREK:SAREK:MARKDUPLICATES:CRAM_QC:SAMTOOLS_STATS (test) [100%] 1 of 1 ✔ -[77/2c8b1b] process > NFCORE_SAREK:SAREK:MARKDUPLICATES:CRAM_QC:MOSDEPTH (test) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:SAMTOOLS_CRAMTOBAM_MARKDUPLICATES - -[f7/499800] process > NFCORE_SAREK:SAREK:PREPARE_RECALIBRATION:BASERECALIBRATOR (test) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:PREPARE_RECALIBRATION:GATHERBQSRREPORTS - -[9d/3d1fff] process > NFCORE_SAREK:SAREK:RECALIBRATE:APPLYBQSR (test) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:RECALIBRATE:MERGE_INDEX_CRAM:MERGE_CRAM - -[cb/fa9dcb] process > NFCORE_SAREK:SAREK:RECALIBRATE:MERGE_INDEX_CRAM:INDEX_CRAM (test) [100%] 1 of 1 ✔ -[19/f075fb] process > NFCORE_SAREK:SAREK:CRAM_QC:SAMTOOLS_STATS (test) [100%] 1 of 1 ✔ -[a9/aca71f] process > NFCORE_SAREK:SAREK:CRAM_QC:MOSDEPTH (test) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:SAMTOOLS_CRAMTOBAM_RECAL - -[14/cb7738] process > NFCORE_SAREK:SAREK:GERMLINE_VARIANT_CALLING:RUN_STRELKA_SINGLE:STRELKA_SINGLE (test) [100%] 1 of 1 ✔ -[- ] process > NFCORE_SAREK:SAREK:GERMLINE_VARIANT_CALLING:RUN_STRELKA_SINGLE:MERGE_STRELKA - -[- ] process > NFCORE_SAREK:SAREK:GERMLINE_VARIANT_CALLING:RUN_STRELKA_SINGLE:MERGE_STRELKA_GENOME - -[- ] process > NFCORE_SAREK:SAREK:TUMOR_ONLY_VARIANT_CALLING:RUN_STRELKA_SINGLE:STRELKA_SINGLE - -[- ] process > NFCORE_SAREK:SAREK:TUMOR_ONLY_VARIANT_CALLING:RUN_STRELKA_SINGLE:MERGE_STRELKA - -[- ] process > NFCORE_SAREK:SAREK:TUMOR_ONLY_VARIANT_CALLING:RUN_STRELKA_SINGLE:MERGE_STRELKA_GENOME - -[- ] process > NFCORE_SAREK:SAREK:PAIR_VARIANT_CALLING:RUN_STRELKA_SOMATIC:STRELKA_SOMATIC - -[- ] process > NFCORE_SAREK:SAREK:PAIR_VARIANT_CALLING:RUN_STRELKA_SOMATIC:MERGE_STRELKA_SNVS - -[- ] process > NFCORE_SAREK:SAREK:PAIR_VARIANT_CALLING:RUN_STRELKA_SOMATIC:MERGE_STRELKA_INDELS - -[3f/68d214] process > NFCORE_SAREK:SAREK:VCF_QC:BCFTOOLS_STATS (test) [100%] 1 of 1 ✔ -[7e/435083] process > NFCORE_SAREK:SAREK:VCF_QC:VCFTOOLS_TSTV_COUNT (test) [100%] 1 of 1 ✔ -[a2/a0c127] process > NFCORE_SAREK:SAREK:VCF_QC:VCFTOOLS_TSTV_QUAL (test) [100%] 1 of 1 ✔ -[98/180e11] process > NFCORE_SAREK:SAREK:VCF_QC:VCFTOOLS_SUMMARY (test) [100%] 1 of 1 ✔ -[40/7f3d8a] process > NFCORE_SAREK:SAREK:CUSTOM_DUMPSOFTWAREVERSIONS (1) [100%] 1 of 1 ✔ -[fa/f4933d] process > NFCORE_SAREK:SAREK:MULTIQC [100%] 1 of 1 ✔ +[85/6b7739] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:BWAMEM1_INDEX (genome.fasta) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:BWAMEM2_INDEX - +[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:DRAGMAP_HASHTABLE - +[22/cf54a8] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:GATK4_CREATESEQUENCEDICTIONARY (genome.fasta) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:MSISENSORPRO_SCAN - +[28/dad25a] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:SAMTOOLS_FAIDX (genome.fasta) [100%] 1 of 1 ✔ +[23/3fe964] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_DBSNP (dbsnp_146.hg38.vcf) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_GERMLINE_RESOURCE - +[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_KNOWN_SNPS - +[14/26e286] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_KNOWN_INDELS (mills_and_1000G.indels.vcf) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:PREPARE_GENOME:TABIX_PON - +[76/04d107] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:CREATE_INTERVALS_BED (genome.interval_list) [100%] 1 of 1 ✔ +[d4/f97174] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:GATK4_INTERVALLISTTOBED (genome) [100%] 1 of 1 ✔ +[70/82ba3c] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:TABIX_BGZIPTABIX_INTERVAL_SPLIT (chr22_1-40001) [100%] 1 of 1 ✔ +[d4/c2d0c4] process > NFCORE_SAREK:SAREK:PREPARE_INTERVALS:TABIX_BGZIPTABIX_INTERVAL_COMBINED (genome) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:SAMTOOLS_VIEW_MAP_MAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:SAMTOOLS_VIEW_UNMAP_UNMAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:SAMTOOLS_VIEW_UNMAP_MAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:SAMTOOLS_VIEW_MAP_UNMAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:SAMTOOLS_MERGE_UNMAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:COLLATE_FASTQ_UNMAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:COLLATE_FASTQ_MAP - +[- ] process > NFCORE_SAREK:SAREK:CONVERT_FASTQ_INPUT:CAT_FASTQ - +[c4/f59e5a] process > NFCORE_SAREK:SAREK:FASTQC (test-test_L1) [100%] 1 of 1 ✔ +[0b/c5a999] process > NFCORE_SAREK:SAREK:FASTQ_ALIGN_BWAMEM_MEM2_DRAGMAP:BWAMEM1_MEM (test) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:FASTQ_ALIGN_BWAMEM_MEM2_DRAGMAP:BWAMEM2_MEM - +[- ] process > NFCORE_SAREK:SAREK:FASTQ_ALIGN_BWAMEM_MEM2_DRAGMAP:DRAGMAP_ALIGN - +[c7/664cd1] process > NFCORE_SAREK:SAREK:BAM_MARKDUPLICATES:GATK4_MARKDUPLICATES (test) [100%] 1 of 1 ✔ +[13/bc73b6] process > NFCORE_SAREK:SAREK:BAM_MARKDUPLICATES:INDEX_MARKDUPLICATES (test) [100%] 1 of 1 ✔ +[2a/99608e] process > NFCORE_SAREK:SAREK:BAM_MARKDUPLICATES:CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS (test) [100%] 1 of 1 ✔ +[f2/0420ca] process > NFCORE_SAREK:SAREK:BAM_MARKDUPLICATES:CRAM_QC_MOSDEPTH_SAMTOOLS:MOSDEPTH (test) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:CRAM_TO_BAM - +[eb/46945a] process > NFCORE_SAREK:SAREK:BAM_BASERECALIBRATOR:GATK4_BASERECALIBRATOR (test) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:BAM_BASERECALIBRATOR:GATK4_GATHERBQSRREPORTS - +[ec/2377d4] process > NFCORE_SAREK:SAREK:BAM_APPLYBQSR:GATK4_APPLYBQSR (test) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:BAM_APPLYBQSR:CRAM_MERGE_INDEX_SAMTOOLS:MERGE_CRAM - +[88/3af664] process > NFCORE_SAREK:SAREK:BAM_APPLYBQSR:CRAM_MERGE_INDEX_SAMTOOLS:INDEX_CRAM (test) [100%] 1 of 1 ✔ +[f4/828fde] process > NFCORE_SAREK:SAREK:CRAM_QC_RECAL:SAMTOOLS_STATS (test) [100%] 1 of 1 ✔ +[fb/a9d66f] process > NFCORE_SAREK:SAREK:CRAM_QC_RECAL:MOSDEPTH (test) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:CRAM_TO_BAM_RECAL - +[ef/026185] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_VARIANT_CALLING_SINGLE_STRELKA:STRELKA_SINGLE (test) [100%] 1 of 1 ✔ +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_VARIANT_CALLING_SINGLE_STRELKA:MERGE_STRELKA - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_VARIANT_CALLING_SINGLE_STRELKA:MERGE_STRELKA_GENOME - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_TUMOR_ONLY_ALL:BAM_VARIANT_CALLING_SINGLE_STRELKA:STRELKA_SINGLE - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_TUMOR_ONLY_ALL:BAM_VARIANT_CALLING_SINGLE_STRELKA:MERGE_STRELKA - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_TUMOR_ONLY_ALL:BAM_VARIANT_CALLING_SINGLE_STRELKA:MERGE_STRELKA_GENOME - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_SOMATIC_ALL:BAM_VARIANT_CALLING_SOMATIC_STRELKA:STRELKA_SOMATIC - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_SOMATIC_ALL:BAM_VARIANT_CALLING_SOMATIC_STRELKA:MERGE_STRELKA_INDELS - +[- ] process > NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_SOMATIC_ALL:BAM_VARIANT_CALLING_SOMATIC_STRELKA:MERGE_STRELKA_SNVS - +[bc/f3f5cf] process > NFCORE_SAREK:SAREK:VCF_QC_BCFTOOLS_VCFTOOLS:BCFTOOLS_STATS (test) [100%] 1 of 1 ✔ +[21/8d4f02] process > NFCORE_SAREK:SAREK:VCF_QC_BCFTOOLS_VCFTOOLS:VCFTOOLS_TSTV_COUNT (test) [100%] 1 of 1 ✔ +[36/957fba] process > NFCORE_SAREK:SAREK:VCF_QC_BCFTOOLS_VCFTOOLS:VCFTOOLS_TSTV_QUAL (test) [100%] 1 of 1 ✔ +[70/a8e064] process > NFCORE_SAREK:SAREK:VCF_QC_BCFTOOLS_VCFTOOLS:VCFTOOLS_SUMMARY (test) [100%] 1 of 1 ✔ +[36/e35b1b] process > NFCORE_SAREK:SAREK:CUSTOM_DUMPSOFTWAREVERSIONS (1) [100%] 1 of 1 ✔ +[3f/3c3356] process > NFCORE_SAREK:SAREK:MULTIQC [100%] 1 of 1 ✔ +-[nf-core/sarek] Pipeline completed successfully- +Completed at: 09-Jun-2023 13:46:31 +Duration : 1m 50s +CPU hours : (a few seconds) +Succeeded : 27 ``` The pipeline comes with a number of possible paths and tools that can be used. -Due to the small test data size, unfortunately not everything can be tested from top-to-bottom, but often is done by utilizing the pipeline's `--step` parameter. Annotation has to tested separatly from the remaining workflow, since we use references for `C.elegans`, while the remaining tests are run on downsampled human data. +Due to the small test data size, unfortunately not everything can be tested from top-to-bottom, but often is done by utilizing the pipeline's `--step` parameter. + +For more extensive testing purpose, we have the `test_cache` profile that contain the same data, but on which the path to the reference and input files can be changed using the `--test_data_base` params. + +Annotation is generally tested separately from the remaining workflow, since we use references for `C.elegans`, while the remaining tests are run on downsampled human data. ```bash -nextflow run nf-core/sarek -r 3.0.1 -profile test, --tools snpeff --step annotation +nextflow run nf-core/sarek -r 3.2.1 -profile test_cache, --outdir results --tools snpeff --step annotation ``` -If you are interested in any of the other tests that are run on every code change or would like to run them yourself, you can take a look at `tests/.yml`. For each entry the respective nextflow command run and the expected output is specified. +If you are interested in any of the other tests that are run on every code change or would like to run them yourself, you can take a look at `tests/.yml`. +For each entry the respective nextflow command run and the expected output is specified. Some of the currently, available test profiles: -| Test profile | Run command | -| :-------------- | :------------------------------------------------------------------------------ | -| annotation | `nextflow run main.nf -profile test,annotation,docker --tools snpeff,vep,merge` | -| no_intervals | `nextflow run main.nf -profile test,no_intervals,docker` | -| targeted | `nextflow run main.nf -profile test,targeted,docker` | -| tools_germline | `nextflow run main.nf -profile test,tools_germline,docker --tools strelka` | -| tools_tumoronly | `nextflow run main.nf -profile test,tools_tumoronly,docker --tools strelka` | -| tools_somatic | `nextflow run main.nf -profile test,tools_somatic,docker --tools strelka` | -| trimming | `nextflow run main.nf -profile test,trim_fastq,docker` | -| umi | `nextflow run main.nf -profile test,umi,docker` | -| use_gatk_spark | `nextflow run main.nf -profile test,use_gatk_spark,docker` | +| Test profile | Run command | +| :-------------- | :------------------------------------------------------------------------------------ | +| annotation | `nextflow run main.nf -profile test_cache,annotation,docker --tools snpeff,vep,merge` | +| no_intervals | `nextflow run main.nf -profile test_cache,no_intervals,docker` | +| targeted | `nextflow run main.nf -profile test_cache,targeted,docker` | +| tools_germline | `nextflow run main.nf -profile test_cache,tools_germline,docker --tools strelka` | +| tools_tumoronly | `nextflow run main.nf -profile test_cache,tools_tumoronly,docker --tools strelka` | +| tools_somatic | `nextflow run main.nf -profile test_cache,tools_somatic,docker --tools strelka` | +| trimming | `nextflow run main.nf -profile test_cache,trim_fastq,docker` | +| umi | `nextflow run main.nf -profile test_cache,umi,docker` | +| use_gatk_spark | `nextflow run main.nf -profile test_cache,use_gatk_spark,docker` | + +If you are interested in any of the other profiles that are used, you can take a look at `conf/test/.config`. ## How can the different steps be used From 51ac097cc6753cfd16465439466dfdd27906f669 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 14:07:38 +0200 Subject: [PATCH 11/47] remove duplicated code --- workflows/sarek.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/workflows/sarek.nf b/workflows/sarek.nf index b11f1d2588..a695989704 100644 --- a/workflows/sarek.nf +++ b/workflows/sarek.nf @@ -147,7 +147,6 @@ fasta_fai = params.fasta_fai ? Channel.fromPath(params.fasta_f germline_resource = params.germline_resource ? Channel.fromPath(params.germline_resource).collect() : Channel.value([]) // Mutec2 does not require a germline resource, so set to optional input known_indels = params.known_indels ? Channel.fromPath(params.known_indels).collect() : Channel.value([]) known_snps = params.known_snps ? Channel.fromPath(params.known_snps).collect() : Channel.value([]) -known_snps = params.known_snps ? Channel.fromPath(params.known_snps).collect() : Channel.value([]) mappability = params.mappability ? Channel.fromPath(params.mappability).collect() : Channel.value([]) pon = params.pon ? Channel.fromPath(params.pon).collect() : Channel.value([]) // PON is optional for Mutect2 (but highly recommended) From 3a4a7f89b0fafcc98fe016ca72d301f31dadd66e Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 14:08:15 +0200 Subject: [PATCH 12/47] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e1412a8c..ff4b758d14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) +- [#1088](https://github.com/nf-core/sarek/pull/1088) - Remove duplicated code ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure From 7fcf0fca9d0b096ea5f8074660828d79138cade0 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 9 Jun 2023 14:09:16 +0200 Subject: [PATCH 13/47] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff4b758d14..174de2aa34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) -- [#1088](https://github.com/nf-core/sarek/pull/1088) - Remove duplicated code +- [#1089](https://github.com/nf-core/sarek/pull/1089) - Remove duplicated code ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure From 7b102356d2b0fe466d5f01a931993f627d247263 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Fri, 9 Jun 2023 20:07:20 +0000 Subject: [PATCH 14/47] Reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. Apparently needed to get the manuael ascat test working. --- subworkflows/local/prepare_genome/main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subworkflows/local/prepare_genome/main.nf b/subworkflows/local/prepare_genome/main.nf index 2a13d5f436..4945282d98 100644 --- a/subworkflows/local/prepare_genome/main.nf +++ b/subworkflows/local/prepare_genome/main.nf @@ -66,26 +66,26 @@ workflow PREPARE_GENOME { // prepare ascat reference files allele_files = ascat_alleles if (params.ascat_alleles && params.ascat_alleles.endsWith('.zip')) { - UNZIP_ALLELES(ascat_alleles.map{ it -> [ [ id:'ascat_alleles' ], it ] }) + UNZIP_ALLELES(ascat_alleles.map{ it -> [[id:it[0].baseName], it]}) allele_files = UNZIP_ALLELES.out.unzipped_archive.map{ it[1] } versions = versions.mix(UNZIP_ALLELES.out.versions) } loci_files = ascat_loci if (params.ascat_loci && params.ascat_loci.endsWith('.zip')) { - UNZIP_LOCI(ascat_loci.map{ it -> [ [ id:'ascat_loci' ], it ] }) + UNZIP_LOCI(ascat_loci.map{ it -> [[id:it[0].baseName], it]}) loci_files = UNZIP_LOCI.out.unzipped_archive.map{ it[1] } versions = versions.mix(UNZIP_LOCI.out.versions) } gc_file = ascat_loci_gc if (params.ascat_loci_gc && params.ascat_loci_gc.endsWith('.zip')) { - UNZIP_GC(ascat_loci_gc.map{ it -> [ [ id:'ascat_loci_gc' ], it ] }) + UNZIP_GC(ascat_loci_gc.map{ it -> [[id:it[0].baseName], it]}) gc_file = UNZIP_GC.out.unzipped_archive.map{ it[1] } versions = versions.mix(UNZIP_GC.out.versions) } rt_file = ascat_loci_rt if (params.ascat_loci_rt && params.ascat_loci_rt.endsWith('.zip')) { - UNZIP_RT(ascat_loci_rt.map{ it -> [ [ id:'ascat_loci_rt' ], it ] }) + UNZIP_RT(ascat_loci_rt.map{ it -> [[id:it[0].baseName], it]}) rt_file = UNZIP_RT.out.unzipped_archive.map{ it[1] } versions = versions.mix(UNZIP_RT.out.versions) } From c3eb66063412e5756dc3ac638fc30574e2289ff3 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Fri, 9 Jun 2023 20:35:04 +0000 Subject: [PATCH 15/47] Updating changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd8aeffc2..6014a6f02b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) - [#1089](https://github.com/nf-core/sarek/pull/1089) - Remove duplicated code +- [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure From fbe605942aec648e61fc61f85a0e139af9993017 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 12 Jun 2023 11:20:25 +0200 Subject: [PATCH 16/47] prepare 3.2.2 --- CHANGELOG.md | 3 ++- nextflow.config | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6014a6f02b..fc47abdca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## dev +## [3.2.2](https://github.com/nf-core/sarek/releases/tag/3.2.2) - Pierikjaure ### Changed - [#1087](https://github.com/nf-core/sarek/pull/1087) - Back to dev - [#1087](https://github.com/nf-core/sarek/pull/1087) - Minor modules update - [#1088](https://github.com/nf-core/sarek/pull/1088) - Replace profile `test` by `test_cache` and add a `test` profile without hidden files +- [#1095](https://github.com/nf-core/sarek/pull/1095) - Prepare release `3.2.2` ### Fixed diff --git a/nextflow.config b/nextflow.config index a695f1d810..4941d3f6ad 100644 --- a/nextflow.config +++ b/nextflow.config @@ -324,7 +324,7 @@ manifest { description = """An open-source analysis pipeline to detect germline or somatic variants from whole genome or targeted sequencing""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '3.3dev' + version = '3.2.2' doi = '10.12688/f1000research.16665.2, 10.5281/zenodo.4063683' } From f5389b6e38dbba438aa32ef85683e84eb5471a89 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Mon, 12 Jun 2023 12:46:24 +0200 Subject: [PATCH 17/47] Update CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc47abdca9..a69650b7fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.2.2](https://github.com/nf-core/sarek/releases/tag/3.2.2) - Pierikjaure +## [3.2.2](https://github.com/nf-core/sarek/releases/tag/3.2.2) - Vuoinesluobbalah + +Vuoinesluobbalah is a lake close to Bierikjávrre. ### Changed From b9030ff8fb7934927cf009322574de7fc8c5537c Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 12 Jun 2023 14:38:48 +0200 Subject: [PATCH 18/47] fix Mutect2 subworkflow bug --- CHANGELOG.md | 1 + .../local/bam_variant_calling_somatic_mutect2/main.nf | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a69650b7fe..072951b7fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) - [#1089](https://github.com/nf-core/sarek/pull/1089) - Remove duplicated code - [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. +- [#1098](https://github.com/nf-core/sarek/pull/1098) - Fix Channel issue in Mutect2 subworkflow [#1094](https://github.com/nf-core/sarek/pull/1094) ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure diff --git a/subworkflows/local/bam_variant_calling_somatic_mutect2/main.nf b/subworkflows/local/bam_variant_calling_somatic_mutect2/main.nf index 893511354e..750cfb8981 100644 --- a/subworkflows/local/bam_variant_calling_somatic_mutect2/main.nf +++ b/subworkflows/local/bam_variant_calling_somatic_mutect2/main.nf @@ -110,9 +110,13 @@ workflow BAM_VARIANT_CALLING_SOMATIC_MUTECT2 { no_intervals: it[0].num_intervals <= 1 } + // Only when using intervals + pileup_table_normal_to_merge = pileup_table_normal_branch.intervals.map{ meta, table -> [ groupKey(meta, meta.num_intervals), table ] }.groupTuple() + pileup_table_tumor_to_merge = pileup_table_tumor_branch.intervals.map{ meta, table -> [ groupKey(meta, meta.num_intervals), table ] }.groupTuple() + // Merge Pileup Summaries - GATHERPILEUPSUMMARIES_NORMAL(GETPILEUPSUMMARIES_NORMAL.out.table.map{ meta, table -> [ groupKey(meta, meta.num_intervals), table ] }.groupTuple(), dict.map{ meta, dict -> [ dict ] }) - GATHERPILEUPSUMMARIES_TUMOR(GETPILEUPSUMMARIES_TUMOR.out.table.map{ meta, table -> [ groupKey(meta, meta.num_intervals), table ] }.groupTuple(), dict.map{ meta, dict -> [ dict ] }) + GATHERPILEUPSUMMARIES_NORMAL(pileup_table_normal_to_merge, dict.map{ meta, dict -> [ dict ] }) + GATHERPILEUPSUMMARIES_TUMOR(pileup_table_tumor_to_merge, dict.map{ meta, dict -> [ dict ] }) // remove no longer necessary field: normal_id, tumor_id, num_intervals pileup_table_normal = Channel.empty().mix(GATHERPILEUPSUMMARIES_NORMAL.out.table, pileup_table_normal_branch.no_intervals) From 15909a4b1b145ec8673cd5276fb57b8463fb665f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 12 Jun 2023 15:16:22 +0200 Subject: [PATCH 19/47] remove duplicate index for deepvariant when no_intervals --- conf/modules/deepvariant.config | 10 +--------- .../local/bam_variant_calling_deepvariant/main.nf | 8 -------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/conf/modules/deepvariant.config b/conf/modules/deepvariant.config index f91e792447..65ed591c4d 100644 --- a/conf/modules/deepvariant.config +++ b/conf/modules/deepvariant.config @@ -22,7 +22,7 @@ process { publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/variant_calling/" }, - pattern: "*vcf.gz", + pattern: "*{vcf.gz,vcf.gz.tbi}", saveAs: { meta.num_intervals > 1 ? null : "deepvariant/${meta.id}/${it}" } ] } @@ -40,12 +40,4 @@ process { ext.prefix = {"${meta.id}.deepvariant.g"} } - withName : 'TABIX_VC_DEEPVARIANT_.*' { - ext.prefix = {"${meta.id}.deepvariant"} - publishDir = [ - mode: params.publish_dir_mode, - path: { "${params.outdir}/variant_calling/deepvariant/${meta.id}/" }, - pattern: "*tbi" - ] - } } diff --git a/subworkflows/local/bam_variant_calling_deepvariant/main.nf b/subworkflows/local/bam_variant_calling_deepvariant/main.nf index c0e07976c5..f763ea3bd2 100644 --- a/subworkflows/local/bam_variant_calling_deepvariant/main.nf +++ b/subworkflows/local/bam_variant_calling_deepvariant/main.nf @@ -1,8 +1,6 @@ include { GATK4_MERGEVCFS as MERGE_DEEPVARIANT_GVCF } from '../../../modules/nf-core/gatk4/mergevcfs/main' include { GATK4_MERGEVCFS as MERGE_DEEPVARIANT_VCF } from '../../../modules/nf-core/gatk4/mergevcfs/main' include { DEEPVARIANT } from '../../../modules/nf-core/deepvariant/main' -include { TABIX_TABIX as TABIX_VC_DEEPVARIANT_GVCF } from '../../../modules/nf-core/tabix/tabix/main' -include { TABIX_TABIX as TABIX_VC_DEEPVARIANT_VCF } from '../../../modules/nf-core/tabix/tabix/main' // Deepvariant: https://github.com/google/deepvariant/issues/510 workflow BAM_VARIANT_CALLING_DEEPVARIANT { @@ -44,10 +42,6 @@ workflow BAM_VARIANT_CALLING_DEEPVARIANT { MERGE_DEEPVARIANT_GVCF(gvcf_to_merge, dict) MERGE_DEEPVARIANT_VCF(vcf_to_merge, dict) - // Only when no_intervals - TABIX_VC_DEEPVARIANT_GVCF(gvcf_out.no_intervals) - TABIX_VC_DEEPVARIANT_VCF(vcf_out.no_intervals) - // Mix intervals and no_intervals channels together gvcf = Channel.empty().mix(MERGE_DEEPVARIANT_GVCF.out.vcf, gvcf_out.no_intervals) // add variantcaller to meta map and remove no longer necessary field: num_intervals @@ -61,8 +55,6 @@ workflow BAM_VARIANT_CALLING_DEEPVARIANT { versions = versions.mix(DEEPVARIANT.out.versions) versions = versions.mix(MERGE_DEEPVARIANT_GVCF.out.versions) versions = versions.mix(MERGE_DEEPVARIANT_VCF.out.versions) - versions = versions.mix(TABIX_VC_DEEPVARIANT_GVCF.out.versions) - versions = versions.mix(TABIX_VC_DEEPVARIANT_VCF.out.versions) emit: gvcf From 66ce8e86b7f131ae0a3a9fd7e36d43c4c3bf02dc Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 12 Jun 2023 15:18:14 +0200 Subject: [PATCH 20/47] update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a69650b7fe..5dc33ce013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085) - [#1089](https://github.com/nf-core/sarek/pull/1089) - Remove duplicated code -- [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. +- [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName +- [#1100](https://github.com/nf-core/sarek/pull/1100) - Remove duplicate index with deepvariant when no_intervals [#1069](https://github.com/nf-core/sarek/pull/1069) ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure From c9eaba9c0f517cf3924182d41304d9d1b9327485 Mon Sep 17 00:00:00 2001 From: Rike Date: Mon, 12 Jun 2023 17:49:24 +0200 Subject: [PATCH 21/47] remove redundant index computation [skip ci] --- subworkflows/local/bam_markduplicates/main.nf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/subworkflows/local/bam_markduplicates/main.nf b/subworkflows/local/bam_markduplicates/main.nf index 89cd1aeb15..2bec370d13 100644 --- a/subworkflows/local/bam_markduplicates/main.nf +++ b/subworkflows/local/bam_markduplicates/main.nf @@ -22,11 +22,8 @@ workflow BAM_MARKDUPLICATES { // RUN MARKUPDUPLICATES GATK4_MARKDUPLICATES(bam, fasta, fasta_fai) - // Index cram - INDEX_MARKDUPLICATES(GATK4_MARKDUPLICATES.out.cram) - // Join with the crai file - cram = GATK4_MARKDUPLICATES.out.cram.join(INDEX_MARKDUPLICATES.out.crai, failOnDuplicate: true, failOnMismatch: true) + cram = GATK4_MARKDUPLICATES.out.cram.join(GATK4_MARKDUPLICATES.out.crai, failOnDuplicate: true, failOnMismatch: true) // QC on CRAM CRAM_QC_MOSDEPTH_SAMTOOLS(cram, fasta, intervals_bed_combined) @@ -37,7 +34,6 @@ workflow BAM_MARKDUPLICATES { // Gather versions of all tools used versions = versions.mix(GATK4_MARKDUPLICATES.out.versions) - versions = versions.mix(INDEX_MARKDUPLICATES.out.versions) versions = versions.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.versions) emit: From 9c3cc8b9cd8bfaa1d696ee2c799bd81bbeea6b6a Mon Sep 17 00:00:00 2001 From: Rike Date: Mon, 12 Jun 2023 17:53:55 +0200 Subject: [PATCH 22/47] update changelog [skip ci] --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac02ed465f..01de035f9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. - [#1098](https://github.com/nf-core/sarek/pull/1098) - Fix Channel issue in Mutect2 subworkflow [#1094](https://github.com/nf-core/sarek/pull/1094) - [#1100](https://github.com/nf-core/sarek/pull/1100) - Remove duplicate index with deepvariant when no_intervals [#1069](https://github.com/nf-core/sarek/pull/1069) +- [#1101](https://github.com/nf-core/sarek/pull/1101) - Remove duplicate index computation for GATK4 Markduplicates & [#1065](https://github.com/nf-core/sarek/issues/1065) +- [#1101](https://github.com/nf-core/sarek/pull/1101) fix GATK4 version for GATK4 MarkduplicatesSpark [#1068](https://github.com/nf-core/sarek/issues/1068) ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure From 0bba02cda9aae684df7db445794ae28a83fb282a Mon Sep 17 00:00:00 2001 From: maxulysse Date: Tue, 13 Jun 2023 17:15:52 +0200 Subject: [PATCH 23/47] remove params.tracedir --- CHANGELOG.md | 1 + nextflow.config | 9 ++++----- nextflow_schema.json | 7 ------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac02ed465f..f59b5d76b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. - [#1098](https://github.com/nf-core/sarek/pull/1098) - Fix Channel issue in Mutect2 subworkflow [#1094](https://github.com/nf-core/sarek/pull/1094) - [#1100](https://github.com/nf-core/sarek/pull/1100) - Remove duplicate index with deepvariant when no_intervals [#1069](https://github.com/nf-core/sarek/pull/1069) +- [#1105](https://github.com/nf-core/sarek/pull/1105) - Remove `params.tracedir` ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure diff --git a/nextflow.config b/nextflow.config index 4941d3f6ad..2a8432ccf0 100644 --- a/nextflow.config +++ b/nextflow.config @@ -99,7 +99,6 @@ params { // Boilerplate options outdir = null - tracedir = "${params.outdir}/pipeline_info" publish_dir_mode = 'copy' email = null email_on_fail = null @@ -302,19 +301,19 @@ singularity.registry = 'quay.io' def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true - file = "${params.tracedir}/execution_timeline_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" } report { enabled = true - file = "${params.tracedir}/execution_report_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" } trace { enabled = true - file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt" + file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" } dag { enabled = true - file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html" } manifest { diff --git a/nextflow_schema.json b/nextflow_schema.json index fa71399258..05e5a83c4f 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -959,13 +959,6 @@ "description": "Custom MultiQC yaml file containing HTML including a methods description.", "fa_icon": "fas fa-cog" }, - "tracedir": { - "type": "string", - "description": "Directory to keep pipeline Nextflow logs and reports.", - "default": "${params.outdir}/pipeline_info", - "fa_icon": "fas fa-cogs", - "hidden": true - }, "validate_params": { "type": "boolean", "description": "Boolean whether to validate parameters against the schema at runtime", From 3215fe406ccc4ea4a7725c06afdea0646bec5a41 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Tue, 13 Jun 2023 19:31:24 +0200 Subject: [PATCH 24/47] add Slack integration --- .github/workflows/awsfulltest.yml | 1 + .github/workflows/awsfulltest_germline.yml | 1 + CHANGELOG.md | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index bd4a18a7ef..9f84f071cb 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -20,6 +20,7 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + hook_url: "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}" workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/somatic_test parameters: | { diff --git a/.github/workflows/awsfulltest_germline.yml b/.github/workflows/awsfulltest_germline.yml index 0674c9297c..03ed3c9ce8 100644 --- a/.github/workflows/awsfulltest_germline.yml +++ b/.github/workflows/awsfulltest_germline.yml @@ -20,6 +20,7 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + hook_url: "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}" workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/germline_test parameters: | { diff --git a/CHANGELOG.md b/CHANGELOG.md index f59b5d76b7..90ea7299d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Vuoinesluobbalah is a lake close to Bierikjávrre. +### Added + +- [#1106](https://github.com/nf-core/sarek/pull/1106) - Add Slack integration to Megatests + ### Changed - [#1087](https://github.com/nf-core/sarek/pull/1087) - Back to dev From a803e27de30302fa764e0847bfd238d2031973d4 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Tue, 13 Jun 2023 19:54:41 +0200 Subject: [PATCH 25/47] it's a params --- .github/workflows/awsfulltest.yml | 2 +- .github/workflows/awsfulltest_germline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 9f84f071cb..9a41b26a0e 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -20,10 +20,10 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - hook_url: "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}" workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/somatic_test parameters: | { + "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/somatic_test" } profiles: test_full,public_aws_ecr diff --git a/.github/workflows/awsfulltest_germline.yml b/.github/workflows/awsfulltest_germline.yml index 03ed3c9ce8..0e9867e4f2 100644 --- a/.github/workflows/awsfulltest_germline.yml +++ b/.github/workflows/awsfulltest_germline.yml @@ -20,10 +20,10 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - hook_url: "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}" workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/germline_test parameters: | { + "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/germline_test" } profiles: test_full_germline,public_aws_ecr From ac637971b024e81a9b160931f3994e6c6d63e8a6 Mon Sep 17 00:00:00 2001 From: Rike Date: Wed, 14 Jun 2023 08:04:20 +0200 Subject: [PATCH 26/47] update md modules --- modules.json | 6 ++--- modules/nf-core/gatk4/markduplicates/main.nf | 26 +++++++++++++------ .../nf-core/gatk4/markduplicatesspark/main.nf | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/modules.json b/modules.json index 97eb570c13..037845d26a 100644 --- a/modules.json +++ b/modules.json @@ -97,7 +97,7 @@ }, "controlfreec/makegraph": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "83ddd598664e16f74e0b5c728e9f81285c37e893", "installed_by": ["modules"] }, "custom/dumpsoftwareversions": { @@ -258,12 +258,12 @@ }, "gatk4/markduplicates": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "0a261469640941da2488e1a5aa023b64db837c70", "installed_by": ["modules"] }, "gatk4/markduplicatesspark": { "branch": "master", - "git_sha": "541811d779026c5d395925895fa5ed35e7216cc0", + "git_sha": "0a261469640941da2488e1a5aa023b64db837c70", "installed_by": ["modules"] }, "gatk4/mergemutectstats": { diff --git a/modules/nf-core/gatk4/markduplicates/main.nf b/modules/nf-core/gatk4/markduplicates/main.nf index cb5c1cc1a1..f4b3f6dcf0 100644 --- a/modules/nf-core/gatk4/markduplicates/main.nf +++ b/modules/nf-core/gatk4/markduplicates/main.nf @@ -2,10 +2,10 @@ process GATK4_MARKDUPLICATES { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.4.0.0" + conda "bioconda::gatk4=4.4.0.0 bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': - 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-d9e7bad0f7fbc8f4458d5c3ab7ffaaf0235b59fb:f857e2d6cc88d35580d01cf39e0959a68b83c1d9-0': + 'biocontainers/mulled-v2-d9e7bad0f7fbc8f4458d5c3ab7ffaaf0235b59fb:f857e2d6cc88d35580d01cf39e0959a68b83c1d9-0' }" input: tuple val(meta), path(bam) @@ -25,7 +25,11 @@ process GATK4_MARKDUPLICATES { script: def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" + prefix = task.ext.prefix ?: "${meta.id}.bam" + + // If the extension is CRAM, then change it to BAM + prefix_bam = prefix.tokenize('.')[-1] == 'cram' ? "${prefix.substring(0, prefix.lastIndexOf('.'))}.bam" : prefix + def input_list = bam.collect{"--INPUT $it"}.join(' ') def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : "" @@ -35,23 +39,29 @@ process GATK4_MARKDUPLICATES { } else { avail_mem = (task.memory.mega*0.8).intValue() } + + // Using samtools and not Markduplicates to compress to CRAM speeds up computation: + // https://medium.com/@acarroll.dna/looking-at-trade-offs-in-compression-levels-for-genomics-tools-eec2834e8b94 """ gatk --java-options "-Xmx${avail_mem}M" MarkDuplicates \\ $input_list \\ - --OUTPUT ${prefix} \\ + --OUTPUT ${prefix_bam} \\ --METRICS_FILE ${prefix}.metrics \\ --TMP_DIR . \\ ${reference} \\ $args - - if [[ ${prefix} == *.cram ]]&&[[ -f ${prefix}.bai ]]; then - mv ${prefix}.bai ${prefix}.crai + # If cram files are wished as output, the run samtools for conversion + if [[ ${prefix} == *.cram ]]; then + samtools view -Ch -T ${fasta} -o ${prefix} ${prefix_bam} + rm ${prefix_bam} + samtools index ${prefix} fi cat <<-END_VERSIONS > versions.yml "${task.process}": gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') END_VERSIONS """ } diff --git a/modules/nf-core/gatk4/markduplicatesspark/main.nf b/modules/nf-core/gatk4/markduplicatesspark/main.nf index f51d598f86..30f4703613 100644 --- a/modules/nf-core/gatk4/markduplicatesspark/main.nf +++ b/modules/nf-core/gatk4/markduplicatesspark/main.nf @@ -2,7 +2,7 @@ process GATK4_MARKDUPLICATES_SPARK { tag "$meta.id" label 'process_high' - conda "bioconda::gatk4=4.3.0.0 conda-forge::openjdk=8.0.312" + conda "bioconda::gatk4=4.4.0.0 conda-forge::openjdk=8.0.312" container "nf-core/gatk:4.4.0.0" input: From 401cf54bf1a7a29d68d99c4a7fd2857d9ccc32d8 Mon Sep 17 00:00:00 2001 From: Friederike Hanssen Date: Wed, 14 Jun 2023 08:14:07 +0200 Subject: [PATCH 27/47] Revert wrong checksum change --- modules.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules.json b/modules.json index 037845d26a..6732e1901e 100644 --- a/modules.json +++ b/modules.json @@ -97,7 +97,7 @@ }, "controlfreec/makegraph": { "branch": "master", - "git_sha": "83ddd598664e16f74e0b5c728e9f81285c37e893", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "custom/dumpsoftwareversions": { From 07a4d5f56201eade0818c0046b7ecd0a918ca538 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 14 Jun 2023 06:43:33 +0000 Subject: [PATCH 28/47] [automated] Fix linting with Prettier --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7434963a5..9d43d69ca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,6 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1101](https://github.com/nf-core/sarek/pull/1101) - Fix GATK4 version for GATK4 MarkduplicatesSpark [#1068](https://github.com/nf-core/sarek/issues/1068) - [#1105](https://github.com/nf-core/sarek/pull/1105) - Remove `params.tracedir` - ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure Pierikjaure is a previous spelling of Bierikjávrre. From 8be49d99b720d9366a449b5b44a8646cfe9a60a5 Mon Sep 17 00:00:00 2001 From: Rike Date: Wed, 14 Jun 2023 09:06:58 +0200 Subject: [PATCH 29/47] remove unnecessary import --- subworkflows/local/bam_markduplicates/main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/subworkflows/local/bam_markduplicates/main.nf b/subworkflows/local/bam_markduplicates/main.nf index 2bec370d13..f64d30be74 100644 --- a/subworkflows/local/bam_markduplicates/main.nf +++ b/subworkflows/local/bam_markduplicates/main.nf @@ -6,7 +6,6 @@ include { CRAM_QC_MOSDEPTH_SAMTOOLS } from '../cram_qc_mosdepth_samtools/main' include { GATK4_MARKDUPLICATES } from '../../../modules/nf-core/gatk4/markduplicates/main' -include { SAMTOOLS_INDEX as INDEX_MARKDUPLICATES } from '../../../modules/nf-core/samtools/index/main' workflow BAM_MARKDUPLICATES { take: From c940acaa6f3eb563ccad7a77c6c5909171e4deee Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 14 Jun 2023 13:11:31 +0200 Subject: [PATCH 30/47] update config --- conf/public_aws_ecr.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/public_aws_ecr.config b/conf/public_aws_ecr.config index 335909060a..75ff69f723 100644 --- a/conf/public_aws_ecr.config +++ b/conf/public_aws_ecr.config @@ -8,8 +8,9 @@ ---------------------------------------------------------------------------------------- */ -docker.registry = 'public.ecr.aws' -podman.registry = 'public.ecr.aws' +docker.registry = 'public.ecr.aws' +podman.registry = 'public.ecr.aws' +singularity.registry = 'public.ecr.aws' process { withName: 'ADD_INFO_TO_VCF' { From 6ead4a9a4ca5ae38e636c77143fa499ac469b487 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 14 Jun 2023 13:14:18 +0200 Subject: [PATCH 31/47] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d43d69ca5..a2c532ead8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. ### Added - [#1106](https://github.com/nf-core/sarek/pull/1106) - Add Slack integration to Megatests +- [#1107](https://github.com/nf-core/sarek/pull/1107) - Add `singularity.registry` to `public_aws_ecr` ### Changed From 91c8c89581d72280bb3e5484376c21c1260cf38a Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 14 Jun 2023 15:50:56 +0200 Subject: [PATCH 32/47] refactor prefix definition for vcf files --- CHANGELOG.md | 1 + conf/modules/annotate.config | 8 ++++---- conf/modules/freebayes.config | 2 +- conf/modules/joint_germline.config | 2 +- conf/modules/modules.config | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c532ead8..d1e313c9d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1101](https://github.com/nf-core/sarek/pull/1101) - Remove duplicate index computation for GATK4 Markduplicates & [#1065](https://github.com/nf-core/sarek/issues/1065) - [#1101](https://github.com/nf-core/sarek/pull/1101) - Fix GATK4 version for GATK4 MarkduplicatesSpark [#1068](https://github.com/nf-core/sarek/issues/1068) - [#1105](https://github.com/nf-core/sarek/pull/1105) - Remove `params.tracedir` +- [#1108](https://github.com/nf-core/sarek/pull/1108) - Refactor bad prefix definition for vcf files [#938](https://github.com/nf-core/sarek/issues/938) ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure diff --git a/conf/modules/annotate.config b/conf/modules/annotate.config index 6e9355d2fb..482f69b4e8 100644 --- a/conf/modules/annotate.config +++ b/conf/modules/annotate.config @@ -18,7 +18,7 @@ process { // SNPEFF if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('merge'))) { withName: 'SNPEFF_SNPEFF' { - ext.prefix = { "${vcf.baseName.minus(".vcf")}_snpEff" } + ext.prefix = { vcf.name - ".vcf" + "_snpEff" } ext.args = '-nodownload -canon -v' if (!params.snpeff_cache && !params.download_cache) container = { params.snpeff_genome ? "docker.io/nfcore/snpeff:${params.snpeff_version}.${params.snpeff_genome}" : "docker.io/nfcore/snpeff:${params.snpeff_version}.${params.genome}" } publishDir = [ @@ -46,7 +46,7 @@ process { (params.vep_custom_args) ?: '' ].join(' ').trim() } // If just VEP: _VEP.ann.vcf - ext.prefix = { "${vcf.baseName.minus(".vcf")}_VEP.ann" } + ext.prefix = { vcf.name - ".vcf" + "_VEP.ann" } if (!params.vep_cache && !params.download_cache) container = { params.vep_genome ? "docker.io/nfcore/vep:${params.vep_version}.${params.vep_genome}" : "docker.io/nfcore/vep:${params.vep_version}.${params.genome}" } publishDir = [ [ @@ -67,14 +67,14 @@ process { if (params.tools && params.tools.split(',').contains('merge')) { withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:VCF_ANNOTATE_MERGE:ENSEMBLVEP_VEP" { // If merge: Output file will have format *_snpEff_VEP.ann.vcf, *_snpEff_VEP.ann.json or *_snpEff_VEP.ann.tab - ext.prefix = { "${vcf.baseName.minus(".ann.vcf")}_VEP.ann" } + ext.prefix = { vcf.name - ".ann.vcf" + "_VEP.ann" } } } // ALL ANNOTATION TOOLS if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('vep') || params.tools.split(',').contains('merge'))) { withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:.*:(TABIX_BGZIPTABIX|TABIX_TABIX)" { - ext.prefix = { input.baseName.minus(".vcf") } + ext.prefix = { input.name - ".vcf" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/annotation/${meta.variantcaller}/${meta.id}/" }, diff --git a/conf/modules/freebayes.config b/conf/modules/freebayes.config index 5243bf699f..35064a36a3 100644 --- a/conf/modules/freebayes.config +++ b/conf/modules/freebayes.config @@ -35,7 +35,7 @@ process { } withName: 'BCFTOOLS_SORT' { - ext.prefix = { meta.num_intervals <= 1 ? "${meta.id}.freebayes" : "${vcf.minus("vcf")}.sort" } + ext.prefix = { meta.num_intervals <= 1 ? meta.id + ".freebayes" : vcf.name - ".vcf" + ".sort" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/variant_calling/" }, diff --git a/conf/modules/joint_germline.config b/conf/modules/joint_germline.config index ac2674bbb3..55ddc787f0 100644 --- a/conf/modules/joint_germline.config +++ b/conf/modules/joint_germline.config @@ -32,7 +32,7 @@ process { if (params.tools && params.tools.contains('haplotypecaller') && params.joint_germline) { withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_GATK:BCFTOOLS_SORT' { - ext.prefix = { "${vcf.baseName.minus("vcf")}sort" } + ext.prefix = { vcf.name - ".vcf" + "sort" } publishDir = [ enabled: false ] diff --git a/conf/modules/modules.config b/conf/modules/modules.config index 0c36169370..faa24f2923 100644 --- a/conf/modules/modules.config +++ b/conf/modules/modules.config @@ -90,7 +90,7 @@ process { // VCF withName: 'BCFTOOLS_STATS' { ext.when = { !(params.skip_tools && params.skip_tools.split(',').contains('bcftools')) } - ext.prefix = { "${vcf.baseName.minus(".vcf")}" } + ext.prefix = { vcf.name - ".vcf" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/reports/bcftools/${meta.variantcaller}/${meta.id}/" }, @@ -100,7 +100,7 @@ process { withName: 'VCFTOOLS_.*' { ext.when = { !(params.skip_tools && params.skip_tools.split(',').contains('vcftools')) } - ext.prefix = { "${variant_file.baseName.minus(".vcf")}" } + ext.prefix = { vcf.name - ".vcf" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/reports/vcftools/${meta.variantcaller}/${meta.id}/" }, From 806f3d6c66124bf9b41611698da53c113bff9054 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 14 Jun 2023 16:00:08 +0200 Subject: [PATCH 33/47] baseName, not name --- conf/modules/annotate.config | 6 +++--- conf/modules/joint_germline.config | 2 +- conf/modules/modules.config | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/modules/annotate.config b/conf/modules/annotate.config index 482f69b4e8..8ee07c22eb 100644 --- a/conf/modules/annotate.config +++ b/conf/modules/annotate.config @@ -18,7 +18,7 @@ process { // SNPEFF if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('merge'))) { withName: 'SNPEFF_SNPEFF' { - ext.prefix = { vcf.name - ".vcf" + "_snpEff" } + ext.prefix = { vcf.baseName - ".vcf" + "_snpEff" } ext.args = '-nodownload -canon -v' if (!params.snpeff_cache && !params.download_cache) container = { params.snpeff_genome ? "docker.io/nfcore/snpeff:${params.snpeff_version}.${params.snpeff_genome}" : "docker.io/nfcore/snpeff:${params.snpeff_version}.${params.genome}" } publishDir = [ @@ -46,7 +46,7 @@ process { (params.vep_custom_args) ?: '' ].join(' ').trim() } // If just VEP: _VEP.ann.vcf - ext.prefix = { vcf.name - ".vcf" + "_VEP.ann" } + ext.prefix = { vcf.baseName - ".vcf" + "_VEP.ann" } if (!params.vep_cache && !params.download_cache) container = { params.vep_genome ? "docker.io/nfcore/vep:${params.vep_version}.${params.vep_genome}" : "docker.io/nfcore/vep:${params.vep_version}.${params.genome}" } publishDir = [ [ @@ -67,7 +67,7 @@ process { if (params.tools && params.tools.split(',').contains('merge')) { withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:VCF_ANNOTATE_MERGE:ENSEMBLVEP_VEP" { // If merge: Output file will have format *_snpEff_VEP.ann.vcf, *_snpEff_VEP.ann.json or *_snpEff_VEP.ann.tab - ext.prefix = { vcf.name - ".ann.vcf" + "_VEP.ann" } + ext.prefix = { vcf.baseName - ".ann.vcf" + "_VEP.ann" } } } diff --git a/conf/modules/joint_germline.config b/conf/modules/joint_germline.config index 55ddc787f0..7427b0da41 100644 --- a/conf/modules/joint_germline.config +++ b/conf/modules/joint_germline.config @@ -32,7 +32,7 @@ process { if (params.tools && params.tools.contains('haplotypecaller') && params.joint_germline) { withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_GATK:BCFTOOLS_SORT' { - ext.prefix = { vcf.name - ".vcf" + "sort" } + ext.prefix = { vcf.baseName - ".vcf" + "sort" } publishDir = [ enabled: false ] diff --git a/conf/modules/modules.config b/conf/modules/modules.config index faa24f2923..f8facd1d8d 100644 --- a/conf/modules/modules.config +++ b/conf/modules/modules.config @@ -90,7 +90,7 @@ process { // VCF withName: 'BCFTOOLS_STATS' { ext.when = { !(params.skip_tools && params.skip_tools.split(',').contains('bcftools')) } - ext.prefix = { vcf.name - ".vcf" } + ext.prefix = { vcf.baseName - ".vcf" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/reports/bcftools/${meta.variantcaller}/${meta.id}/" }, @@ -100,7 +100,7 @@ process { withName: 'VCFTOOLS_.*' { ext.when = { !(params.skip_tools && params.skip_tools.split(',').contains('vcftools')) } - ext.prefix = { vcf.name - ".vcf" } + ext.prefix = { vcf.baseName - ".vcf" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/reports/vcftools/${meta.variantcaller}/${meta.id}/" }, From febf7d894431788eb33cf420671cc2f8ffcdd962 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Wed, 14 Jun 2023 16:13:41 +0200 Subject: [PATCH 34/47] Update conf/modules/joint_germline.config Co-authored-by: Robert Syme --- conf/modules/joint_germline.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules/joint_germline.config b/conf/modules/joint_germline.config index 7427b0da41..c03e629b22 100644 --- a/conf/modules/joint_germline.config +++ b/conf/modules/joint_germline.config @@ -32,7 +32,7 @@ process { if (params.tools && params.tools.contains('haplotypecaller') && params.joint_germline) { withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_GATK:BCFTOOLS_SORT' { - ext.prefix = { vcf.baseName - ".vcf" + "sort" } + ext.prefix = { vcf.baseName - ".vcf" + ".sort" } publishDir = [ enabled: false ] From 62767075ba90e4b10affb4daf1022189868bdcf2 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 14 Jun 2023 17:05:18 +0200 Subject: [PATCH 35/47] variant_file not vcf --- conf/modules/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules/modules.config b/conf/modules/modules.config index f8facd1d8d..57276bc157 100644 --- a/conf/modules/modules.config +++ b/conf/modules/modules.config @@ -100,7 +100,7 @@ process { withName: 'VCFTOOLS_.*' { ext.when = { !(params.skip_tools && params.skip_tools.split(',').contains('vcftools')) } - ext.prefix = { vcf.baseName - ".vcf" } + ext.prefix = { variant_file.baseName - ".vcf" } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/reports/vcftools/${meta.variantcaller}/${meta.id}/" }, From 37be36de69052ad7258d8abf168d80eb499b4f84 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 15 Jun 2023 07:23:05 +0000 Subject: [PATCH 36/47] fix bcftools mpileup --- conf/modules/mpileup.config | 15 +++++++-- docs/output.md | 32 +++++++++---------- .../local/bam_variant_calling_mpileup/main.nf | 18 +++++------ tests/test_mpileup.yml | 29 +++++++++++++---- 4 files changed, 59 insertions(+), 35 deletions(-) diff --git a/conf/modules/mpileup.config b/conf/modules/mpileup.config index e9c86e6ee5..aaa7e79ae0 100644 --- a/conf/modules/mpileup.config +++ b/conf/modules/mpileup.config @@ -31,14 +31,23 @@ process { ext.args3 = "-i 'count(GT==\"RR\")==0'" // only report non homozygous reference variants publishDir = [ mode: params.publish_dir_mode, - path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" }, + path: { "${params.outdir}/variant_calling/bcftools/${meta.id}/" }, pattern: "*{vcf.gz,vcf.gz.tbi}", saveAs: { meta.num_intervals > 1 ? null : it } ] } + withName: 'MERGE_BCFTOOLS_MPILEUP' { + ext.prefix = {"${meta.id}.bcftools"} + publishDir = [ + mode: params.publish_dir_mode, + path: { "${params.outdir}/variant_calling/bcftools/${meta.id}/" }, + pattern: "*{vcf.gz,vcf.gz.tbi}" + ] + } + withName: 'SAMTOOLS_MPILEUP' { - ext.when = { params.tools && (params.tools.split(',').contains('controlfreec') || params.tools.split(',').contains('mpileup')) } + ext.when = { params.tools && params.tools.split(',').contains('controlfreec') } publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" }, @@ -49,7 +58,7 @@ process { } // PAIR_VARIANT_CALLING - if (params.tools && (params.tools.split(',').contains('controlfreec') || params.tools.split(',').contains('mpileup'))) { + if (params.tools && params.tools.split(',').contains('controlfreec')) { withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_VARIANT_CALLING_MPILEUP:SAMTOOLS_MPILEUP' { ext.prefix = { meta.num_intervals <= 1 ? "${meta.id}.normal" : "${meta.id}_${intervals.simpleName}.normal" } } diff --git a/docs/output.md b/docs/output.md index 1a033e9f2b..4bde2dcbc5 100644 --- a/docs/output.md +++ b/docs/output.md @@ -30,7 +30,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d - [FreeBayes](#freebayes) - [GATK HaplotypeCaller](#gatk-haplotypecaller) - [GATK Mutect2](#gatk-mutect2) - - [samtools mpileup](#samtools-mpileup) + - [bcftools](#bcftools) - [Strelka2](#strelka2) - [Structural Variants](#structural-variants) - [Manta](#manta) @@ -279,6 +279,21 @@ If some results from a variant caller do not appear here, please check out the ` For single nucleotide variants (SNVs) and small indels, multiple tools are available for normal (germline), tumor-only, and tumor-normal (somatic) paired data. For a list of the appropriate tool(s) for the data and sequencing type at hand, please check [here](usage.md#which-tool). +#### bcftools mpileup + +[bcftools mpileup](https://samtools.github.io/bcftools/bcftools.html#mpileup) generates pileup of a CRAM file, followed by [bcftools call](https://samtools.github.io/bcftools/bcftools.html#call) and filtered with `-i 'count(GT==\"RR\")==0`. +For further reading and documentation see the [bcftools manual](https://samtools.github.io/bcftools/howtos/variant-calling.html). + +
+Output files for all samples + +**Output directory: `{outdir}/variantcalling/bcftools//`** + +- `.bcftools.vcf.gz` and `.bcftools.vcf.gz.tbi` + - VCF with tabix index + +
+ #### DeepVariant [DeepVariant](https://github.com/google/deepvariant) is a deep learning-based variant caller that takes aligned reads, produces pileup image tensors from them, classifies each tensor using a convolutional neural network and finally reports the results in a standard VCF or gVCF file. For further documentation take a look [here](https://github.com/google/deepvariant/tree/r1.4/docs). @@ -384,21 +399,6 @@ Files created: -#### samtools mpileup - -[samtools mpileup](https://www.htslib.org/doc/samtools-mpileup.html) generates pileup of a CRAM file. -For further reading and documentation see the [samtools manual](https://www.htslib.org/doc/samtools-mpileup.html). - -
-Output files for all samples - -**Output directory: `{outdir}/variantcalling/mpileup//`** - -- `.pileup.gz` - - The pileup format is a text-based format for summarizing the base calls of aligned reads to a reference sequence. Alignment records are grouped by sample (`SM`) identifiers in `@RG` header lines. - -
- #### Strelka2 [Strelka2](https://github.com/Illumina/strelka) is a fast and accurate small variant caller optimized for analysis of germline variation in small cohorts and somatic variation in tumor/normal sample pairs. For further reading and documentation see the [Strelka2 user guide](https://github.com/Illumina/strelka/blob/master/docs/userGuide/README.md). If [Strelka2](https://github.com/Illumina/strelka) is used for somatic variant calling and [Manta](https://github.com/Illumina/manta) is also specified in tools, the output candidate indels from [Manta](https://github.com/Illumina/manta) are used according to [Strelka Best Practices](https://github.com/Illumina/strelka/blob/master/docs/userGuide/README.md#somatic-configuration-example). diff --git a/subworkflows/local/bam_variant_calling_mpileup/main.nf b/subworkflows/local/bam_variant_calling_mpileup/main.nf index 63f111c95a..772753d781 100644 --- a/subworkflows/local/bam_variant_calling_mpileup/main.nf +++ b/subworkflows/local/bam_variant_calling_mpileup/main.nf @@ -1,7 +1,7 @@ -include { CAT_CAT as CAT_MPILEUP } from '../../../modules/nf-core/cat/cat/main' -include { BCFTOOLS_MPILEUP } from '../../../modules/nf-core/bcftools/mpileup/main' -include { SAMTOOLS_MPILEUP } from '../../../modules/nf-core/samtools/mpileup/main' -include { GATK4_MERGEVCFS } from '../../../modules/nf-core/gatk4/mergevcfs/main' +include { CAT_CAT as CAT_MPILEUP } from '../../../modules/nf-core/cat/cat/main' +include { BCFTOOLS_MPILEUP } from '../../../modules/nf-core/bcftools/mpileup/main' +include { SAMTOOLS_MPILEUP } from '../../../modules/nf-core/samtools/mpileup/main' +include { GATK4_MERGEVCFS as MERGE_BCFTOOLS_MPILEUP } from '../../../modules/nf-core/gatk4/mergevcfs/main' workflow BAM_VARIANT_CALLING_MPILEUP { @@ -44,20 +44,20 @@ workflow BAM_VARIANT_CALLING_MPILEUP { // Merge VCF vcf_to_merge = vcf_mpileup.intervals.map{ meta, vcf -> [ groupKey(meta, meta.num_intervals), vcf ] }.groupTuple() - GATK4_MERGEVCFS(vcf_to_merge, dict) + MERGE_BCFTOOLS_MPILEUP(vcf_to_merge, dict) // Mix intervals and no_intervals channels together mpileup = CAT_MPILEUP.out.file_out.mix(mpileup_samtools.no_intervals) // add variantcaller to meta map and remove no longer necessary field: num_intervals - .map{ meta, mpileup -> [ meta - meta.subMap('num_intervals') + [ variantcaller:'mpileup' ], mpileup ] } - vcf = GATK4_MERGEVCFS.out.vcf.mix(vcf_mpileup.no_intervals) + .map{ meta, mpileup -> [ meta - meta.subMap('num_intervals') + [ variantcaller:'samtools' ], mpileup ] } + vcf = MERGE_BCFTOOLS_MPILEUP.out.vcf.mix(vcf_mpileup.no_intervals) // add variantcaller to meta map and remove no longer necessary field: num_intervals - .map{ meta, vcf -> [ meta - meta.subMap('num_intervals') + [ variantcaller:'mpileup' ], vcf ] } + .map{ meta, vcf -> [ meta - meta.subMap('num_intervals') + [ variantcaller:'bcftools' ], vcf ] } versions = versions.mix(SAMTOOLS_MPILEUP.out.versions) versions = versions.mix(BCFTOOLS_MPILEUP.out.versions) versions = versions.mix(CAT_MPILEUP.out.versions) - versions = versions.mix(GATK4_MERGEVCFS.out.versions) + versions = versions.mix(MERGE_BCFTOOLS_MPILEUP.out.versions) emit: mpileup diff --git a/tests/test_mpileup.yml b/tests/test_mpileup.yml index 6ac744f434..7171743158 100644 --- a/tests/test_mpileup.yml +++ b/tests/test_mpileup.yml @@ -5,8 +5,12 @@ - mpileup files: - path: results/multiqc + - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz + md5sum: 07c3eb437131bfa5ec036e7da039ad2f + - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz.tbi + md5sum: d828e532551df740c98e9b1c0582cf31 - path: results/variant_calling/mpileup/sample2/sample2.tumor.mpileup.gz - # binary changes md5sums on reruns + should_exist: false - path: results/mpileup should_exist: false - name: Run variant calling on tumor_only sample to test mpileup without intervals @@ -23,8 +27,12 @@ md5sum: f3dac01ea66b95fe477446fde2d31489 - path: results/no_intervals.bed.gz.tbi md5sum: f3dac01ea66b95fe477446fde2d31489 - - path: results/variant_calling/mpileup/sample2/sample2.tumor.mpileup.gz - # binary changes md5sums on reruns + - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz + md5sum: 07c3eb437131bfa5ec036e7da039ad2f + - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz.tbi + md5sum: d828e532551df740c98e9b1c0582cf31 + - path: results/variant_calling/mpileup/ + should_exist: false - path: results/mpileup should_exist: false - name: Run variant calling on germline sample to test mpileup @@ -34,8 +42,12 @@ - mpileup files: - path: results/multiqc - - path: results/variant_calling/mpileup/sample1/sample1.normal.mpileup.gz - # binary changes md5sums on reruns + - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz + md5sum: 152f5925eef9a05f96a66b805ff7f931 + - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz.tbi + md5sum: 277b1f3c0f32e80ed736d0d927c94a62 + - path: results/variant_calling/mpileup/ + should_exist: false - path: results/mpileup should_exist: false - name: Run variant calling on germline sample to test mpileup without intervals @@ -52,7 +64,10 @@ md5sum: f3dac01ea66b95fe477446fde2d31489 - path: results/no_intervals.bed.gz.tbi md5sum: f3dac01ea66b95fe477446fde2d31489 - - path: results/variant_calling/mpileup/sample1/sample1.normal.mpileup.gz - # binary changes md5sums on reruns + - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz + md5sum: 152f5925eef9a05f96a66b805ff7f931 + - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz.tbi + md5sum: 277b1f3c0f32e80ed736d0d927c94a62 + - path: results/variant_calling/mpileup/ - path: results/mpileup should_exist: false From 59923eb088de677ebd61526d054f0eb4daecc421 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 15 Jun 2023 07:28:48 +0000 Subject: [PATCH 37/47] consolidate docs --- docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/output.md b/docs/output.md index 4bde2dcbc5..1992c32a6d 100644 --- a/docs/output.md +++ b/docs/output.md @@ -279,7 +279,7 @@ If some results from a variant caller do not appear here, please check out the ` For single nucleotide variants (SNVs) and small indels, multiple tools are available for normal (germline), tumor-only, and tumor-normal (somatic) paired data. For a list of the appropriate tool(s) for the data and sequencing type at hand, please check [here](usage.md#which-tool). -#### bcftools mpileup +#### bcftools [bcftools mpileup](https://samtools.github.io/bcftools/bcftools.html#mpileup) generates pileup of a CRAM file, followed by [bcftools call](https://samtools.github.io/bcftools/bcftools.html#call) and filtered with `-i 'count(GT==\"RR\")==0`. For further reading and documentation see the [bcftools manual](https://samtools.github.io/bcftools/howtos/variant-calling.html). From 5213603bca81c7162f6fac5039eebf0840a3207d Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 15 Jun 2023 07:30:07 +0000 Subject: [PATCH 38/47] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac02ed465f..44c98dc08b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ Vuoinesluobbalah is a lake close to Bierikjávrre. - [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName. - [#1098](https://github.com/nf-core/sarek/pull/1098) - Fix Channel issue in Mutect2 subworkflow [#1094](https://github.com/nf-core/sarek/pull/1094) - [#1100](https://github.com/nf-core/sarek/pull/1100) - Remove duplicate index with deepvariant when no_intervals [#1069](https://github.com/nf-core/sarek/pull/1069) - +- [#1109](https://github.com/nf-core/sarek/pull/1109) - Fix `mpileup` for variantcalling: only `bcftools` run and file publishing ## [3.2.1](https://github.com/nf-core/sarek/releases/tag/3.2.1) - Pierikjaure Pierikjaure is a previous spelling of Bierikjávrre. From 387f4f4e5801fe7aa4bf3d4461484c0724834344 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 15 Jun 2023 07:34:46 +0000 Subject: [PATCH 39/47] also fix annotation for mpileup --- .../local/bam_variant_calling_tumor_only_all/main.nf | 5 ++++- workflows/sarek.nf | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf b/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf index 8507899366..3831151309 100644 --- a/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf +++ b/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf @@ -41,6 +41,7 @@ workflow BAM_VARIANT_CALLING_TUMOR_ONLY_ALL { //TODO: Temporary until the if's can be removed and printing to terminal is prevented with "when" in the modules.config vcf_freebayes = Channel.empty() vcf_manta = Channel.empty() + vcf_mpileup = Channel.empty() vcf_mutect2 = Channel.empty() vcf_strelka = Channel.empty() vcf_tiddit = Channel.empty() @@ -53,7 +54,7 @@ workflow BAM_VARIANT_CALLING_TUMOR_ONLY_ALL { fasta, intervals ) - + vcf_mpileup = BAM_VARIANT_CALLING_MPILEUP.out.vcf versions = versions.mix(BAM_VARIANT_CALLING_MPILEUP.out.versions) } @@ -172,6 +173,7 @@ workflow BAM_VARIANT_CALLING_TUMOR_ONLY_ALL { vcf_freebayes, vcf_manta, vcf_mutect2, + vcf_mpileup, vcf_strelka, vcf_tiddit ) @@ -180,6 +182,7 @@ workflow BAM_VARIANT_CALLING_TUMOR_ONLY_ALL { vcf_all vcf_freebayes vcf_manta + vcf_mpileup vcf_mutect2 vcf_strelka vcf_tiddit diff --git a/workflows/sarek.nf b/workflows/sarek.nf index a695989704..e4bb247771 100644 --- a/workflows/sarek.nf +++ b/workflows/sarek.nf @@ -1011,7 +1011,7 @@ workflow SAREK { vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_GERMLINE_ALL.out.vcf_manta) vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_GERMLINE_ALL.out.vcf_strelka) vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_GERMLINE_ALL.out.vcf_tiddit) - // vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_GERMLINE_ALL.out.vcf_mpileup) // Not annotated? + vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_GERMLINE_ALL.out.vcf_mpileup) vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_TUMOR_ONLY_ALL.out.vcf_all) vcf_to_annotate = vcf_to_annotate.mix(BAM_VARIANT_CALLING_SOMATIC_ALL.out.vcf_all) From 0ed285e7559678db33f2880243989025927e401d Mon Sep 17 00:00:00 2001 From: Friederike Hanssen Date: Thu, 15 Jun 2023 09:36:58 +0200 Subject: [PATCH 40/47] Update subworkflows/local/bam_variant_calling_mpileup/main.nf Co-authored-by: Maxime U Garcia --- subworkflows/local/bam_variant_calling_mpileup/main.nf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/subworkflows/local/bam_variant_calling_mpileup/main.nf b/subworkflows/local/bam_variant_calling_mpileup/main.nf index 772753d781..137fbc4cf6 100644 --- a/subworkflows/local/bam_variant_calling_mpileup/main.nf +++ b/subworkflows/local/bam_variant_calling_mpileup/main.nf @@ -1,9 +1,7 @@ -include { CAT_CAT as CAT_MPILEUP } from '../../../modules/nf-core/cat/cat/main' +include { CAT_CAT as CAT_MPILEUP } from '../../../modules/nf-core/cat/cat/main' include { BCFTOOLS_MPILEUP } from '../../../modules/nf-core/bcftools/mpileup/main' include { SAMTOOLS_MPILEUP } from '../../../modules/nf-core/samtools/mpileup/main' include { GATK4_MERGEVCFS as MERGE_BCFTOOLS_MPILEUP } from '../../../modules/nf-core/gatk4/mergevcfs/main' - - workflow BAM_VARIANT_CALLING_MPILEUP { take: cram // channel: [mandatory] [ meta, cram, crai ] From 434507442ee6e8c7c32a39716fe8c1a8b318aedc Mon Sep 17 00:00:00 2001 From: Friederike Hanssen Date: Thu, 15 Jun 2023 09:38:40 +0200 Subject: [PATCH 41/47] Update subworkflows/local/bam_variant_calling_tumor_only_all/main.nf --- subworkflows/local/bam_variant_calling_tumor_only_all/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf b/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf index 3831151309..54f2d877f2 100644 --- a/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf +++ b/subworkflows/local/bam_variant_calling_tumor_only_all/main.nf @@ -41,7 +41,7 @@ workflow BAM_VARIANT_CALLING_TUMOR_ONLY_ALL { //TODO: Temporary until the if's can be removed and printing to terminal is prevented with "when" in the modules.config vcf_freebayes = Channel.empty() vcf_manta = Channel.empty() - vcf_mpileup = Channel.empty() + vcf_mpileup = Channel.empty() vcf_mutect2 = Channel.empty() vcf_strelka = Channel.empty() vcf_tiddit = Channel.empty() From 62aa7ee4187cad8d00ae0e07b9aec55189611e0d Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 15 Jun 2023 09:42:07 +0200 Subject: [PATCH 42/47] Update .github/workflows/pytest-workflow_release.yml --- .github/workflows/pytest-workflow_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-workflow_release.yml b/.github/workflows/pytest-workflow_release.yml index a2abde262b..27d5130486 100644 --- a/.github/workflows/pytest-workflow_release.yml +++ b/.github/workflows/pytest-workflow_release.yml @@ -1,7 +1,7 @@ name: pytest-workflow-release # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: + pull_request: branches: [master] release: types: [published] From fcd5b64685205857770ba7593d8898ae27058522 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 15 Jun 2023 07:56:28 +0000 Subject: [PATCH 43/47] fix more publishing issues with mpileup --- conf/modules/mpileup.config | 10 ++-------- tests/test_controlfreec.yml | 10 +++++----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/conf/modules/mpileup.config b/conf/modules/mpileup.config index aaa7e79ae0..9ef259daaf 100644 --- a/conf/modules/mpileup.config +++ b/conf/modules/mpileup.config @@ -17,10 +17,7 @@ process { withName: 'CAT_MPILEUP' { publishDir = [ - enabled: true, - mode: params.publish_dir_mode, - path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" }, - pattern: "*{mpileup.gz}" + enabled: false ] } @@ -49,10 +46,7 @@ process { withName: 'SAMTOOLS_MPILEUP' { ext.when = { params.tools && params.tools.split(',').contains('controlfreec') } publishDir = [ - mode: params.publish_dir_mode, - path: { "${params.outdir}/variant_calling/mpileup/${meta.id}/" }, - pattern: "*mpileup.gz", - saveAs: { meta.num_intervals > 1 ? null : it } + enabled: false ] } diff --git a/tests/test_controlfreec.yml b/tests/test_controlfreec.yml index 9c84ad8fd0..32393aa501 100644 --- a/tests/test_controlfreec.yml +++ b/tests/test_controlfreec.yml @@ -47,9 +47,9 @@ - path: results/variant_calling/controlfreec/sample4_vs_sample3/sample4_vs_sample3_ratio.png # binary changes md5sums on reruns - path: results/variant_calling/mpileup/sample4_vs_sample3/sample4_vs_sample3.normal.mpileup.gz - # binary changes md5sums on reruns + should_exist: false - path: results/variant_calling/mpileup/sample4_vs_sample3/sample4_vs_sample3.tumor.mpileup.gz - # binary changes md5sums on reruns + should_exist: false - path: results/cnvkit should_exist: false - name: Run variant calling on somatic samples with controlfreec without intervals @@ -98,9 +98,9 @@ - path: results/variant_calling/controlfreec/sample4_vs_sample3/sample4_vs_sample3_sample.cpn md5sum: d41d8cd98f00b204e9800998ecf8427e - path: results/variant_calling/mpileup/sample4_vs_sample3/sample4_vs_sample3.normal.mpileup.gz - # binary changes md5sums on reruns + should_exist: false - path: results/variant_calling/mpileup/sample4_vs_sample3/sample4_vs_sample3.tumor.mpileup.gz - # binary changes md5sums on reruns + should_exist: false - path: results/controlfreec should_exist: false - path: results/mpileup @@ -143,7 +143,7 @@ - path: results/variant_calling/controlfreec/sample2/sample2_sample.cpn md5sum: d41d8cd98f00b204e9800998ecf8427e - path: results/variant_calling/mpileup/sample2/sample2.tumor.mpileup.gz - # binary changes md5sums on reruns + should_exist: false - path: results/controlfreec should_exist: false - path: results/mpileup From e3d80cf9c4bb4f5f45821b95fcab524cb3129dc0 Mon Sep 17 00:00:00 2001 From: Friederike Hanssen Date: Thu, 15 Jun 2023 10:03:52 +0200 Subject: [PATCH 44/47] Update tests/test_mpileup.yml --- tests/test_mpileup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_mpileup.yml b/tests/test_mpileup.yml index 7171743158..dae0dc11e7 100644 --- a/tests/test_mpileup.yml +++ b/tests/test_mpileup.yml @@ -69,5 +69,6 @@ - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz.tbi md5sum: 277b1f3c0f32e80ed736d0d927c94a62 - path: results/variant_calling/mpileup/ + should_exist: false - path: results/mpileup should_exist: false From 8fb0b537bdb70a2b9a1fb3c56c274cf364a13902 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Thu, 15 Jun 2023 09:14:19 +0000 Subject: [PATCH 45/47] variable md5sums --- tests/test_mpileup.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_mpileup.yml b/tests/test_mpileup.yml index dae0dc11e7..9ebfad4890 100644 --- a/tests/test_mpileup.yml +++ b/tests/test_mpileup.yml @@ -28,9 +28,9 @@ - path: results/no_intervals.bed.gz.tbi md5sum: f3dac01ea66b95fe477446fde2d31489 - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz - md5sum: 07c3eb437131bfa5ec036e7da039ad2f + # binary changes md5sums on reruns - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz.tbi - md5sum: d828e532551df740c98e9b1c0582cf31 + # binary changes md5sums on reruns - path: results/variant_calling/mpileup/ should_exist: false - path: results/mpileup @@ -43,9 +43,9 @@ files: - path: results/multiqc - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz - md5sum: 152f5925eef9a05f96a66b805ff7f931 + # binary changes md5sums on reruns - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz.tbi - md5sum: 277b1f3c0f32e80ed736d0d927c94a62 + # binary changes md5sums on reruns - path: results/variant_calling/mpileup/ should_exist: false - path: results/mpileup @@ -65,9 +65,9 @@ - path: results/no_intervals.bed.gz.tbi md5sum: f3dac01ea66b95fe477446fde2d31489 - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz - md5sum: 152f5925eef9a05f96a66b805ff7f931 + # binary changes md5sums on reruns - path: results/variant_calling/bcftools/sample1/sample1.bcftools.vcf.gz.tbi - md5sum: 277b1f3c0f32e80ed736d0d927c94a62 + # binary changes md5sums on reruns - path: results/variant_calling/mpileup/ should_exist: false - path: results/mpileup From 2e2397d07dcf5428e0e7cb25fa802a668487e27e Mon Sep 17 00:00:00 2001 From: Friederike Hanssen Date: Thu, 15 Jun 2023 11:51:49 +0200 Subject: [PATCH 46/47] Update tests/test_mpileup.yml Co-authored-by: Maxime U Garcia --- tests/test_mpileup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mpileup.yml b/tests/test_mpileup.yml index 9ebfad4890..7cff9dad23 100644 --- a/tests/test_mpileup.yml +++ b/tests/test_mpileup.yml @@ -6,9 +6,9 @@ files: - path: results/multiqc - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz - md5sum: 07c3eb437131bfa5ec036e7da039ad2f + # binary changes md5sums on reruns - path: results/variant_calling/bcftools/sample2/sample2.bcftools.vcf.gz.tbi - md5sum: d828e532551df740c98e9b1c0582cf31 + # binary changes md5sums on reruns - path: results/variant_calling/mpileup/sample2/sample2.tumor.mpileup.gz should_exist: false - path: results/mpileup From 702e3bc0e3717ce4a52b741e31a4a259d0a1564d Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 15 Jun 2023 14:37:40 +0200 Subject: [PATCH 47/47] Apply suggestions from code review --- .github/workflows/pytest-workflow_release.yml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/pytest-workflow_release.yml b/.github/workflows/pytest-workflow_release.yml index 27d5130486..3c02664cd7 100644 --- a/.github/workflows/pytest-workflow_release.yml +++ b/.github/workflows/pytest-workflow_release.yml @@ -34,29 +34,13 @@ jobs: fail-fast: false matrix: tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] - profile: ["docker", "singularity", "conda"] + profile: ["docker", "singularity"] TEST_DATA_BASE: - "test-datasets/data" NXF_VER: - "23.04.0" - "latest-everything" exclude: - - profile: "conda" - tags: concatenate_vcfs - - profile: "conda" - tags: deepvariant - - profile: "conda" - tags: haplotypecaller - - profile: "conda" - tags: merge - - profile: "conda" - tags: snpeff - - profile: "conda" - tags: umi - - profile: "conda" - tags: validation_checks - - profile: "conda" - tags: vep - profile: "singularity" tags: concatenate_vcfs - profile: "singularity"