From 8ce34a40589137b75b65dfe8bb334c9b94f1d6c8 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 20 Mar 2024 10:28:29 +1300 Subject: [PATCH 01/19] Added liftoff (#5209) * Added liftoff * Added -db arg --- modules/nf-core/liftoff/environment.yml | 9 ++ modules/nf-core/liftoff/main.nf | 66 ++++++++++++ modules/nf-core/liftoff/meta.yml | 71 ++++++++++++ modules/nf-core/liftoff/tests/main.nf.test | 102 ++++++++++++++++++ .../nf-core/liftoff/tests/main.nf.test.snap | 96 +++++++++++++++++ modules/nf-core/liftoff/tests/nextflow.config | 5 + modules/nf-core/liftoff/tests/tags.yml | 2 + 7 files changed, 351 insertions(+) create mode 100644 modules/nf-core/liftoff/environment.yml create mode 100644 modules/nf-core/liftoff/main.nf create mode 100644 modules/nf-core/liftoff/meta.yml create mode 100644 modules/nf-core/liftoff/tests/main.nf.test create mode 100644 modules/nf-core/liftoff/tests/main.nf.test.snap create mode 100644 modules/nf-core/liftoff/tests/nextflow.config create mode 100644 modules/nf-core/liftoff/tests/tags.yml diff --git a/modules/nf-core/liftoff/environment.yml b/modules/nf-core/liftoff/environment.yml new file mode 100644 index 00000000000..8761c9bf73e --- /dev/null +++ b/modules/nf-core/liftoff/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "liftoff" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::liftoff=1.6.3" diff --git a/modules/nf-core/liftoff/main.nf b/modules/nf-core/liftoff/main.nf new file mode 100644 index 00000000000..4db5da2f462 --- /dev/null +++ b/modules/nf-core/liftoff/main.nf @@ -0,0 +1,66 @@ +process LIFTOFF { + tag "$meta.id" + label 'process_high' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/liftoff:1.6.3--pyhdfd78af_0': + 'biocontainers/liftoff:1.6.3--pyhdfd78af_0' }" + + input: + tuple val(meta), path(target_fa) + path ref_fa, name: 'ref_assembly.fa' + path ref_annotation + path ref_db + + output: + tuple val(meta), path("${prefix}.gff3") , emit: gff3 + tuple val(meta), path("*.polished.gff3") , emit: polished_gff3, optional: true + tuple val(meta), path("*.unmapped.txt") , emit: unmapped_txt + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def arg_g = ref_annotation ? "-g $ref_annotation" : '' + def arg_db = ref_db ? "-db $ref_db" : '' + prefix = task.ext.prefix ?: "${meta.id}" + """ + liftoff \\ + $arg_g \\ + $arg_db \\ + -p $task.cpus \\ + -o "${prefix}.gff3" \\ + -u "${prefix}.unmapped.txt" \\ + $args \\ + $target_fa \\ + ref_assembly.fa + + mv \\ + "${prefix}.gff3_polished" \\ + "${prefix}.polished.gff3" \\ + || echo "-polish is absent" + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + liftoff: \$(liftoff --version 2> /dev/null) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + def touch_polished = args.contains('-polish') ? "touch ${prefix}.polished.gff3" : '' + """ + touch "${prefix}.gff3" + touch "${prefix}.unmapped.txt" + $touch_polished + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + liftoff: \$(liftoff --version 2> /dev/null) + END_VERSIONS + """ +} diff --git a/modules/nf-core/liftoff/meta.yml b/modules/nf-core/liftoff/meta.yml new file mode 100644 index 00000000000..10e502c150b --- /dev/null +++ b/modules/nf-core/liftoff/meta.yml @@ -0,0 +1,71 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "liftoff" +description: | + Uses Liftoff to accurately map annotations in GFF or GTF between assemblies of the same, + or closely-related species +keywords: + - genome + - annotation + - gff3 + - gtf + - liftover +tools: + - "liftoff": + description: | + Liftoff is a tool that accurately maps annotations in GFF or GTF between assemblies of the same, + or closely-related species + homepage: "https://github.com/agshumate/Liftoff" + documentation: "https://github.com/agshumate/Liftoff" + tool_dev_url: "https://github.com/agshumate/Liftoff" + doi: "10.1093/bioinformatics/bty191" + licence: ["GPL v3 License"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'test' ]` + - target_fa: + type: file + description: Target assembly in fasta format + pattern: "*.{fsa,fa,fasta}" + - ref_fa: + type: file + description: Reference assembly in fasta format + pattern: "*.{fsa,fa,fasta}" + - ref_annotation: + type: file + description: Reference assembly annotations in gtf or gff3 format + pattern: "*.{gtf,gff3}" + - ref_db: + type: file + description: | + Name of feature database; if not specified, the -g argument must + be provided and a database will be built automatically +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'test' ]` + - gff3: + type: file + description: Lifted annotations for the target assembly in gff3 format + pattern: "*.gff3" + - polished_gff3: + type: file + description: Polished lifted annotations for the target assembly in gff3 format + pattern: "*.polished.gff3" + - unmapped_txt: + type: file + description: List of unmapped reference annotations + pattern: "*.unmapped.txt" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@GallVp" +maintainers: + - "@GallVp" diff --git a/modules/nf-core/liftoff/tests/main.nf.test b/modules/nf-core/liftoff/tests/main.nf.test new file mode 100644 index 00000000000..fc0f56719ae --- /dev/null +++ b/modules/nf-core/liftoff/tests/main.nf.test @@ -0,0 +1,102 @@ +nextflow_process { + + name "Test Process LIFTOFF" + script "../main.nf" + process "LIFTOFF" + config "./nextflow.config" + + tag "modules" + tag "modules_nfcore" + tag "nf-core/gunzip" + tag "liftoff" + tag "gunzip" + + test("homo_sapiens-genome_21_fasta-genome_1_fasta-genome_1_gtf") { + + setup { + run("GUNZIP") { + script "../../../nf-core/gunzip" + + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['homo_sapiens']['genome']['genome_1_fasta'], checkIfExists: true) + ] + """ + } + } + } + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) + ] + input[1] = GUNZIP.out.gunzip.map { meta, file -> file } + input[2] = [ + file(params.test_data['homo_sapiens']['genome']['genome_1_gtf'], checkIfExists: true) + ] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.unmapped_txt).match("unmapped_txt") }, + { assert file(process.out.gff3[0][1]).text.contains("chr21\tLiftoff\texon\t34608061\t34608118\t.\t+\t.") }, + { assert file(process.out.polished_gff3[0][1]).text.contains("chr21\tLiftoff\texon\t34608061\t34608118\t.\t+\t.") }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + + } + + test("homo_sapiens-genome_21_fasta-genome_1_fasta-genome_1_gtf-stub") { + options '-stub' + + setup { + run("GUNZIP") { + script "../../../nf-core/gunzip" + + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['homo_sapiens']['genome']['genome_1_fasta'], checkIfExists: true) + ] + """ + } + } + } + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) + ] + input[1] = GUNZIP.out.gunzip.map { meta, file -> file } + input[2] = [ + file(params.test_data['homo_sapiens']['genome']['genome_1_gtf'], checkIfExists: true) + ] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/liftoff/tests/main.nf.test.snap b/modules/nf-core/liftoff/tests/main.nf.test.snap new file mode 100644 index 00000000000..f6064467e33 --- /dev/null +++ b/modules/nf-core/liftoff/tests/main.nf.test.snap @@ -0,0 +1,96 @@ +{ + "unmapped_txt": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.unmapped.txt:md5,7391d10df6e15db356b084c9af5259e4" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2023-12-01T13:57:40.748507" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,205d9c609e7fe27d8199550d842bdce8" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2023-12-01T13:57:40.752414" + }, + "homo_sapiens-genome_21_fasta-genome_1_fasta-genome_1_gtf-stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.gff3:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.polished.gff3:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + [ + { + "id": "test" + }, + "test.unmapped.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + "versions.yml:md5,205d9c609e7fe27d8199550d842bdce8" + ], + "gff3": [ + [ + { + "id": "test" + }, + "test.gff3:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "polished_gff3": [ + [ + { + "id": "test" + }, + "test.polished.gff3:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "unmapped_txt": [ + [ + { + "id": "test" + }, + "test.unmapped.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,205d9c609e7fe27d8199550d842bdce8" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T09:15:25.661428" + } +} \ No newline at end of file diff --git a/modules/nf-core/liftoff/tests/nextflow.config b/modules/nf-core/liftoff/tests/nextflow.config new file mode 100644 index 00000000000..f35ef801698 --- /dev/null +++ b/modules/nf-core/liftoff/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: LIFTOFF { + ext.args = '-polish' + } +} diff --git a/modules/nf-core/liftoff/tests/tags.yml b/modules/nf-core/liftoff/tests/tags.yml new file mode 100644 index 00000000000..4d0adb62c6c --- /dev/null +++ b/modules/nf-core/liftoff/tests/tags.yml @@ -0,0 +1,2 @@ +liftoff: + - "modules/nf-core/liftoff/**" From 5ed10bdf45859615a71ef7ff975a26bb321f4ea6 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 20 Mar 2024 11:59:35 +1300 Subject: [PATCH 02/19] Added gt/gff3validator (#4718) * Added gt/gff3validator * Make error handling a bit more intuitive * Fixed a problem with info echo and simplified test assertions --------- Co-authored-by: Leon Rauschning <99650940+lrauschning@users.noreply.github.com> --- .../nf-core/gt/gff3validator/environment.yml | 9 ++ modules/nf-core/gt/gff3validator/main.nf | 61 ++++++++++ modules/nf-core/gt/gff3validator/meta.yml | 49 ++++++++ .../gt/gff3validator/tests/main.nf.test | 98 ++++++++++++++++ .../gt/gff3validator/tests/main.nf.test.snap | 107 ++++++++++++++++++ .../gt/gff3validator/tests/nextflow.config | 5 + .../nf-core/gt/gff3validator/tests/tags.yml | 2 + 7 files changed, 331 insertions(+) create mode 100644 modules/nf-core/gt/gff3validator/environment.yml create mode 100644 modules/nf-core/gt/gff3validator/main.nf create mode 100644 modules/nf-core/gt/gff3validator/meta.yml create mode 100644 modules/nf-core/gt/gff3validator/tests/main.nf.test create mode 100644 modules/nf-core/gt/gff3validator/tests/main.nf.test.snap create mode 100644 modules/nf-core/gt/gff3validator/tests/nextflow.config create mode 100644 modules/nf-core/gt/gff3validator/tests/tags.yml diff --git a/modules/nf-core/gt/gff3validator/environment.yml b/modules/nf-core/gt/gff3validator/environment.yml new file mode 100644 index 00000000000..ea57ebe06a8 --- /dev/null +++ b/modules/nf-core/gt/gff3validator/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "gt_gff3validator" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::genometools-genometools=1.6.5" diff --git a/modules/nf-core/gt/gff3validator/main.nf b/modules/nf-core/gt/gff3validator/main.nf new file mode 100644 index 00000000000..8c6899cafa3 --- /dev/null +++ b/modules/nf-core/gt/gff3validator/main.nf @@ -0,0 +1,61 @@ +process GT_GFF3VALIDATOR { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/genometools-genometools:1.6.5--py310h3db02ab_0': + 'biocontainers/genometools-genometools:1.6.5--py310h3db02ab_0' }" + + input: + tuple val(meta), path(gff3) + + output: + tuple val(meta), path('*.success.log') , emit: success_log , optional: true + tuple val(meta), path('*.error.log') , emit: error_log , optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + gt \\ + gff3validator \\ + "$gff3" \\ + > "${prefix}.stdout" \\ + 2> >(tee "${prefix}.stderr" >&2) \\ + || echo "Errors from gt-gff3validator printed to ${prefix}.error.log" + + if grep -q "input is valid GFF3" "${prefix}.stdout"; then + echo "Validation successful..." + # emit stdout to the success output channel + mv \\ + "${prefix}.stdout" \\ + "${prefix}.success.log" + else + echo "Validation failed..." + # emit stderr to the error output channel + mv \\ + "${prefix}.stderr" \\ + "${prefix}.error.log" + fi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + genometools: \$(gt --version | head -1 | sed 's/gt (GenomeTools) //') + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch "${prefix}.success.log" + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + genometools: \$(gt --version | head -1 | sed 's/gt (GenomeTools) //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/gt/gff3validator/meta.yml b/modules/nf-core/gt/gff3validator/meta.yml new file mode 100644 index 00000000000..3322faf90cf --- /dev/null +++ b/modules/nf-core/gt/gff3validator/meta.yml @@ -0,0 +1,49 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "gt_gff3validator" +description: "GenomeTools gt-gff3validator utility to strictly validate a GFF3 file" +keywords: + - genome + - gff3 + - annotation + - validation +tools: + - "gt": + description: "The GenomeTools genome analysis system" + homepage: "https://genometools.org/index.html" + documentation: "https://genometools.org/documentation.html" + tool_dev_url: "https://github.com/genometools/genometools" + doi: "10.1109/TCBB.2013.68" + licence: ["ISC"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'test' ]` + - gff3: + type: file + description: Input gff3 file + pattern: "*.{gff,gff3}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'test' ]` + - success_log: + type: file + description: Log file for successful validation + pattern: "*.success.log" + - error_log: + type: file + description: Log file for failed validation + pattern: "*.error.log" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@GallVp" +maintainers: + - "@GallVp" diff --git a/modules/nf-core/gt/gff3validator/tests/main.nf.test b/modules/nf-core/gt/gff3validator/tests/main.nf.test new file mode 100644 index 00000000000..766e3e7425d --- /dev/null +++ b/modules/nf-core/gt/gff3validator/tests/main.nf.test @@ -0,0 +1,98 @@ +nextflow_process { + + name "Test Process GT_GFF3VALIDATOR" + script "../main.nf" + process "GT_GFF3VALIDATOR" + + tag "modules" + tag "modules_nfcore" + tag "gt" + tag "gt/gff3validator" + tag "gt/gff3" + + test("sarscov2-gff3-valid") { + + config './nextflow.config' + + setup { + run("GT_GFF3") { + script "../../../../nf-core/gt/gff3" + + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['sarscov2']['genome']['genome_gff3'], checkIfExists: true) + ] + """ + } + } + } + + when { + process { + """ + input[0] = GT_GFF3.out.gt_gff3 + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert path(process.out.success_log[0][1]).text.contains("input is valid GFF3") }, + { assert process.out.error_log == [] } + ) + } + + } + + test("sarscov2-gff3-stub") { + + options '-stub' + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['sarscov2']['genome']['genome_gff3'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("sarscov2-gff3-invalid") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['genome']['genome_gff3'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert process.out.success_log == [] }, + { assert path(process.out.error_log[0][1]).text.contains("gt gff3validator: error:") } + ) + } + + } +} diff --git a/modules/nf-core/gt/gff3validator/tests/main.nf.test.snap b/modules/nf-core/gt/gff3validator/tests/main.nf.test.snap new file mode 100644 index 00000000000..ccc83f90f51 --- /dev/null +++ b/modules/nf-core/gt/gff3validator/tests/main.nf.test.snap @@ -0,0 +1,107 @@ +{ + "sarscov2-gff3-stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.success.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,5927673eb73a8c22408643d224414215" + ], + "error_log": [ + + ], + "success_log": [ + [ + { + "id": "test" + }, + "test.success.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,5927673eb73a8c22408643d224414215" + ] + } + ], + "timestamp": "2024-01-11T09:29:13.14468" + }, + "sarscov2-gff3-invalid": { + "content": [ + { + "0": [ + + ], + "1": [ + [ + { + "id": "test" + }, + "test.error.log:md5,c5d16b263a87072a13cca44fd811b8e2" + ] + ], + "2": [ + "versions.yml:md5,5927673eb73a8c22408643d224414215" + ], + "error_log": [ + [ + { + "id": "test" + }, + "test.error.log:md5,c5d16b263a87072a13cca44fd811b8e2" + ] + ], + "success_log": [ + + ], + "versions": [ + "versions.yml:md5,5927673eb73a8c22408643d224414215" + ] + } + ], + "timestamp": "2023-11-29T11:09:23.708792" + }, + "sarscov2-gff3-valid": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.success.log:md5,b11ca5c18c865fc808ea0fef0b07da30" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,5927673eb73a8c22408643d224414215" + ], + "error_log": [ + + ], + "success_log": [ + [ + { + "id": "test" + }, + "test.success.log:md5,b11ca5c18c865fc808ea0fef0b07da30" + ] + ], + "versions": [ + "versions.yml:md5,5927673eb73a8c22408643d224414215" + ] + } + ], + "timestamp": "2023-12-13T10:34:54.485391" + } +} \ No newline at end of file diff --git a/modules/nf-core/gt/gff3validator/tests/nextflow.config b/modules/nf-core/gt/gff3validator/tests/nextflow.config new file mode 100644 index 00000000000..c69915daf5d --- /dev/null +++ b/modules/nf-core/gt/gff3validator/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: GT_GFF3 { + ext.args = '-tidy -retainids' + } +} diff --git a/modules/nf-core/gt/gff3validator/tests/tags.yml b/modules/nf-core/gt/gff3validator/tests/tags.yml new file mode 100644 index 00000000000..98c39ffe080 --- /dev/null +++ b/modules/nf-core/gt/gff3validator/tests/tags.yml @@ -0,0 +1,2 @@ +gt/gff3validator: + - "modules/nf-core/gt/gff3validator/**" From 383a1fa09439aedca02a86c93fa1b0a5d699a7eb Mon Sep 17 00:00:00 2001 From: Andrea Telatin <15690844+telatin@users.noreply.github.com> Date: Wed, 20 Mar 2024 03:14:53 +0000 Subject: [PATCH 03/19] Add module seqfu/stats (#5275) * seqfu/stats * fix test * improve-tests * improve-tests-fixsnap * Update modules/nf-core/seqfu/stats/main.nf Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> * Update modules/nf-core/seqfu/stats/main.nf Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> * Update modules/nf-core/seqfu/stats/tests/main.nf.test Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> * Update modules/nf-core/seqfu/stats/tests/main.nf.test Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> --------- Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/nf-core/seqfu/stats/environment.yml | 9 +++ modules/nf-core/seqfu/stats/main.nf | 51 +++++++++++++ modules/nf-core/seqfu/stats/meta.yml | 57 ++++++++++++++ .../nf-core/seqfu/stats/tests/main.nf.test | 75 +++++++++++++++++++ .../seqfu/stats/tests/main.nf.test.snap | 56 ++++++++++++++ modules/nf-core/seqfu/stats/tests/tags.yml | 2 + 6 files changed, 250 insertions(+) create mode 100644 modules/nf-core/seqfu/stats/environment.yml create mode 100644 modules/nf-core/seqfu/stats/main.nf create mode 100644 modules/nf-core/seqfu/stats/meta.yml create mode 100644 modules/nf-core/seqfu/stats/tests/main.nf.test create mode 100644 modules/nf-core/seqfu/stats/tests/main.nf.test.snap create mode 100644 modules/nf-core/seqfu/stats/tests/tags.yml diff --git a/modules/nf-core/seqfu/stats/environment.yml b/modules/nf-core/seqfu/stats/environment.yml new file mode 100644 index 00000000000..abd60be65b4 --- /dev/null +++ b/modules/nf-core/seqfu/stats/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "seqfu_stats" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::seqfu=1.20.3" diff --git a/modules/nf-core/seqfu/stats/main.nf b/modules/nf-core/seqfu/stats/main.nf new file mode 100644 index 00000000000..0f8bb3e220a --- /dev/null +++ b/modules/nf-core/seqfu/stats/main.nf @@ -0,0 +1,51 @@ +process SEQFU_STATS { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/seqfu:1.20.3--h1eb128b_0': + 'biocontainers/seqfu:1.20.3--h1eb128b_0' }" + + + input: + // stats can get one or more fasta or fastq files + tuple val(meta), path(files) + + output: + tuple val(meta), path("*.tsv") , emit: stats + tuple val(meta), path("*_mqc.txt"), emit: multiqc + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + seqfu \\ + stats \\ + $args \\ + --multiqc ${prefix}_mqc.txt \\ + $files > ${prefix}.tsv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + seqfu: \$(seqfu version) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}_mqc.txt + seqfu stats ${prefix}_mqc.txt > ${prefix}.tsv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + seqfu: \$(samtools --version |& sed '1!d ; s/samtools //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/seqfu/stats/meta.yml b/modules/nf-core/seqfu/stats/meta.yml new file mode 100644 index 00000000000..df6093c5120 --- /dev/null +++ b/modules/nf-core/seqfu/stats/meta.yml @@ -0,0 +1,57 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "seqfu_stats" +description: Statistics for FASTA or FASTQ files +keywords: + - seqfu + - stats + - n50 +tools: + - "seqfu": + description: "Cross-platform compiled suite of tools to manipulate and inspect FASTA and FASTQ files" + homepage: "https://telatin.github.io/seqfu2/" + documentation: "https://telatin.github.io/seqfu2/" + tool_dev_url: "https://github.com/telatin/seqfu2" + doi: "10.3390/bioengineering8050059" + licence: ["GPL v3"] + +input: + # Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + + - files: + type: file + description: One or more FASTA or FASTQ files + pattern: "*.{fasta,fastq,fasta.gz,fastq.gz,fq,fq.gz}" + +output: + #Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + + - stats: + type: file + description: Tab-separated output file with basic sequence statistics. + pattern: "*.{tsv}" + + - multiqc: + type: file + description: MultiQC ready table + pattern: "*.{_mqc.txt}" + +authors: + - "@telatin" +maintainers: + - "@telatin" diff --git a/modules/nf-core/seqfu/stats/tests/main.nf.test b/modules/nf-core/seqfu/stats/tests/main.nf.test new file mode 100644 index 00000000000..d8d7de6049e --- /dev/null +++ b/modules/nf-core/seqfu/stats/tests/main.nf.test @@ -0,0 +1,75 @@ +nextflow_process { + + name "Test Process SEQFU_STATS" + script "../main.nf" + process "SEQFU_STATS" + + tag "modules" + tag "modules_nfcore" + tag "seqfu" + tag "seqfu/stats" + + + test("seqfu stats - faa") { + // test with 1 FAA file (with multiple sequences of different length) + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['sarscov2']['genome']['proteome_fasta'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.stats }, + { assert process.out.multiqc }, + { assert process.out.stats.size() == 1 }, + { assert snapshot(process.out.versions).match("versions-single") }, + { assert snapshot(process.out.stats).match("stats-single") }, + { assert path(process.out.stats.get(0).get(1)).md5 == "26141ef87ad8a6f59a6f283cc0a06fda" } + ) + } + + } + + test("seqfu stats - multiple files") { + // test feeding a mix of files including compressed + when { + process { + """ + input[0] = [ + [ id:'test' ], + [ + file(params.test_data['sarscov2']['genome']['proteome_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['genome_fasta_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.stats }, + { assert process.out.multiqc }, + { assert process.out.stats.size() == 1 }, + { assert path(process.out.versions[0]).readLines()[1].contains('.') }, + { assert snapshot(process.out.stats).match("stats-multi") }, + { assert snapshot(process.out.stats).md5().match("multi-lines") }, + { assert path(process.out.stats[0][1]).readLines()[0] == 'File\t#Seq\tTotal bp\tAvg\tN50\tN75\tN90\tauN\tMin\tMax' }, + { assert path(process.out.multiqc[0][1]).readLines().join('\n').contains('genome.fasta') }, + { assert path(process.out.multiqc[0][1]).readLines().join('\n').contains('proteome.fasta') } + ) + } + + } + +} diff --git a/modules/nf-core/seqfu/stats/tests/main.nf.test.snap b/modules/nf-core/seqfu/stats/tests/main.nf.test.snap new file mode 100644 index 00000000000..8049c827b33 --- /dev/null +++ b/modules/nf-core/seqfu/stats/tests/main.nf.test.snap @@ -0,0 +1,56 @@ +{ + "versions-single": { + "content": [ + [ + "versions.yml:md5,fd11b4665f68f22e7ad4c646ad3c56cd" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T20:21:50.896221522" + }, + "multi-lines": { + "content": "d5ab14ce74939f856edc64c6a8d250d3", + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T20:22:03.775940585" + }, + "stats-single": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.tsv:md5,26141ef87ad8a6f59a6f283cc0a06fda" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T20:21:50.937732021" + }, + "stats-multi": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.tsv:md5,98102573ecf7e3b9e53db1c6e0f02b06" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T20:22:03.681146614" + } +} \ No newline at end of file diff --git a/modules/nf-core/seqfu/stats/tests/tags.yml b/modules/nf-core/seqfu/stats/tests/tags.yml new file mode 100644 index 00000000000..2685ed43339 --- /dev/null +++ b/modules/nf-core/seqfu/stats/tests/tags.yml @@ -0,0 +1,2 @@ +seqfu/stats: + - "modules/nf-core/seqfu/stats/**" From c2342b45bbfc5d6fe19cc4154b38fa57177c67dd Mon Sep 17 00:00:00 2001 From: Kobe Lavaerts <77338340+kobelavaerts@users.noreply.github.com> Date: Wed, 20 Mar 2024 08:27:57 +0100 Subject: [PATCH 04/19] nf-test bases2fastq (#5272) * add nf-test to bases2fastq * add untar tag to tags.yml * added stub * [automated] Fix linting with Prettier --------- Co-authored-by: nf-core-bot Co-authored-by: Joon Klaps <61584065+Joon-Klaps@users.noreply.github.com> --- modules/nf-core/bases2fastq/main.nf | 17 +++ .../nf-core/bases2fastq/tests/main.nf.test | 78 +++++++++++++ .../bases2fastq/tests/main.nf.test.snap | 104 ++++++++++++++++++ modules/nf-core/bases2fastq/tests/tags.yml | 3 + tests/config/pytest_modules.yml | 4 - tests/modules/nf-core/bases2fastq/main.nf | 23 ---- .../nf-core/bases2fastq/nextflow.config | 9 -- tests/modules/nf-core/bases2fastq/test.yml | 17 --- 8 files changed, 202 insertions(+), 53 deletions(-) create mode 100644 modules/nf-core/bases2fastq/tests/main.nf.test create mode 100644 modules/nf-core/bases2fastq/tests/main.nf.test.snap create mode 100644 modules/nf-core/bases2fastq/tests/tags.yml delete mode 100644 tests/modules/nf-core/bases2fastq/main.nf delete mode 100644 tests/modules/nf-core/bases2fastq/nextflow.config delete mode 100644 tests/modules/nf-core/bases2fastq/test.yml diff --git a/modules/nf-core/bases2fastq/main.nf b/modules/nf-core/bases2fastq/main.nf index 44d83102dec..2f12b6d0f62 100644 --- a/modules/nf-core/bases2fastq/main.nf +++ b/modules/nf-core/bases2fastq/main.nf @@ -41,4 +41,21 @@ process BASES2FASTQ { bases2fastq: \$(bases2fastq --version | sed -e "s/bases2fastq version //g") END_VERSIONS """ + + stub: + """ + mkdir output + mkdir output/Samples + mkdir output/Samples/DefaultSample + + touch output/Metrics.csv + touch output/RunManifest.json + touch output/UnassignedSequences.csv + echo | gzip > output/Samples/DefaultSample/DefaultSample_R1.fastq.gz + echo | gzip > output/Samples/DefaultSample/DefaultSample_R2.fastq.gz + touch output/Bases2Fastq-Sim_QC.html + touch output/RunStats.json + touch output/Samples/DefaultSample/DefaultSample_stats.json + touch versions.yml + """ } diff --git a/modules/nf-core/bases2fastq/tests/main.nf.test b/modules/nf-core/bases2fastq/tests/main.nf.test new file mode 100644 index 00000000000..cf91d0c9c5f --- /dev/null +++ b/modules/nf-core/bases2fastq/tests/main.nf.test @@ -0,0 +1,78 @@ +nextflow_process { + + name "Test Process BASES2FASTQ" + script "../main.nf" + process "BASES2FASTQ" + + tag "modules" + tag "modules_nfcore" + tag "bases2fastq" + tag "untar" + + setup { + run("UNTAR") { + script "modules/nf-core/untar/main.nf" + process { + """ + input[0] = Channel.of([ [id: "sim-data", samplesheet: file("https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv", checkIfExists: true)], file("https://github.com/nf-core/test-datasets/raw/demultiplex/testdata/sim-data/sim-data.tar.gz", checkIfExists: true) ]) + """ + } + } + } + + test("sim-data.tar.gz - bcl") { + when { + process { + """ + input[0]= UNTAR.out.untar.map{meta, untar -> [meta, meta.samplesheet, untar]} + + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.metrics, + process.out.generated_run_manifest, + process.out.unassigned, + process.out.sample_fastq, + file(process.out.qc_report[0][1]).name, + file(process.out.run_stats[0][1]).name, + file(process.out.sample_json[0][1]).name + ).match() + } + ) + } + + } + + test("sim-data.tar.gz - bcl -stub") { + options "-stub" + when { + process { + """ + input[0]= UNTAR.out.untar.map{meta, untar -> [meta, meta.samplesheet, untar]} + + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.metrics, + process.out.generated_run_manifest, + process.out.unassigned, + process.out.sample_fastq, + file(process.out.qc_report[0][1]).name, + file(process.out.run_stats[0][1]).name, + file(process.out.sample_json[0][1]).name + ).match() + } + ) + } + } +} diff --git a/modules/nf-core/bases2fastq/tests/main.nf.test.snap b/modules/nf-core/bases2fastq/tests/main.nf.test.snap new file mode 100644 index 00000000000..bbf89247b88 --- /dev/null +++ b/modules/nf-core/bases2fastq/tests/main.nf.test.snap @@ -0,0 +1,104 @@ +{ + "sim-data.tar.gz - bcl -stub": { + "content": [ + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + "Metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + "RunManifest.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + "UnassignedSequences.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + [ + "DefaultSample_R1.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "DefaultSample_R2.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ] + ], + "Bases2Fastq-Sim_QC.html", + "RunStats.json", + "DefaultSample_stats.json" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T16:28:23.376144712" + }, + "sim-data.tar.gz - bcl": { + "content": [ + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + "Metrics.csv:md5,a8094b308b5653071ac029a27733e4a6" + ] + ], + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + "RunManifest.json:md5,a07dce8ee25c2a6f9355b677c26b53e2" + ] + ], + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + "UnassignedSequences.csv:md5,11c1693830ce941b8cfb8d2431a59097" + ] + ], + [ + [ + { + "id": "sim-data", + "samplesheet": "/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv" + }, + [ + "DefaultSample_R1.fastq.gz:md5,eeec01b3de76c2b849b0f08b18081a2d", + "DefaultSample_R2.fastq.gz:md5,ccbd16b893d2a069b91d5e4284109868" + ] + ] + ], + "Bases2Fastq-Sim_QC.html", + "RunStats.json", + "DefaultSample_stats.json" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T16:20:46.509235687" + } +} \ No newline at end of file diff --git a/modules/nf-core/bases2fastq/tests/tags.yml b/modules/nf-core/bases2fastq/tests/tags.yml new file mode 100644 index 00000000000..1643c9798b7 --- /dev/null +++ b/modules/nf-core/bases2fastq/tests/tags.yml @@ -0,0 +1,3 @@ +bases2fastq: + - "modules/nf-core/bases2fastq/**" + - "modules/nf-core/untar/**" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 3834021e04d..d9733192abe 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -100,10 +100,6 @@ bandage/image: barrnap: - modules/nf-core/barrnap/** - tests/modules/nf-core/barrnap/** -bases2fastq: - - modules/nf-core/untar/** - - modules/nf-core/bases2fastq/** - - tests/modules/nf-core/bases2fastq/** basicpy: - modules/nf-core/basicpy/** - tests/modules/nf-core/basicpy/** diff --git a/tests/modules/nf-core/bases2fastq/main.nf b/tests/modules/nf-core/bases2fastq/main.nf deleted file mode 100644 index d9fdac02715..00000000000 --- a/tests/modules/nf-core/bases2fastq/main.nf +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { BASES2FASTQ } from '../../../../modules/nf-core/bases2fastq/main.nf' -include { UNTAR } from '../../../../modules/nf-core/untar/main.nf' - -workflow test_bases2fastq { - - input = Channel.value([ - [ id:'sim-data' ], // meta map - file("https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/testdata/sim-data/RunManifest.csv", checkIfExists: true), - ]) - - ch_input = input.join( - UNTAR ( [ - [ id:'sim-data' ], - file("https://github.com/nf-core/test-datasets/raw/demultiplex/testdata/sim-data/sim-data.tar.gz", checkIfExists: true) - ]).untar - ) - - BASES2FASTQ ( ch_input ) -} diff --git a/tests/modules/nf-core/bases2fastq/nextflow.config b/tests/modules/nf-core/bases2fastq/nextflow.config deleted file mode 100644 index 2977ef7761a..00000000000 --- a/tests/modules/nf-core/bases2fastq/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: UNTAR { - publishDir = [ enabled: false ] - } - -} diff --git a/tests/modules/nf-core/bases2fastq/test.yml b/tests/modules/nf-core/bases2fastq/test.yml deleted file mode 100644 index 08d78381756..00000000000 --- a/tests/modules/nf-core/bases2fastq/test.yml +++ /dev/null @@ -1,17 +0,0 @@ -- name: "bases2fastq" - command: nextflow run ./tests/modules/nf-core/bases2fastq -entry test_bases2fastq -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/bases2fastq/nextflow.config - tags: - - "bases2fastq" - files: - - path: output/bases2fastq/output/Bases2Fastq-Sim_QC.html - - path: output/bases2fastq/output/Metrics.csv - md5sum: a8094b308b5653071ac029a27733e4a6 - - path: output/bases2fastq/output/RunManifest.json - - path: output/bases2fastq/output/RunStats.json - - path: output/bases2fastq/output/UnassignedSequences.csv - md5sum: 11c1693830ce941b8cfb8d2431a59097 - - path: output/bases2fastq/output/Samples/DefaultSample/DefaultSample_R1.fastq.gz - md5sum: 831c90ea31eff881e825cda691da77ae - - path: output/bases2fastq/output/Samples/DefaultSample/DefaultSample_R2.fastq.gz - md5sum: e6c1f7ec00910ef195ff3d427c7a9f23 - - path: output/bases2fastq/output/Samples/DefaultSample/DefaultSample_stats.json From f42872b165d4242b6b2d8ea6b2d1035c64acf403 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Wed, 20 Mar 2024 08:53:05 +0100 Subject: [PATCH 05/19] update paths for VEP (#5281) update paths --- modules/nf-core/ensemblvep/vep/tests/main.nf.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/ensemblvep/vep/tests/main.nf.test b/modules/nf-core/ensemblvep/vep/tests/main.nf.test index f072dcabd75..e7cbebd481e 100644 --- a/modules/nf-core/ensemblvep/vep/tests/main.nf.test +++ b/modules/nf-core/ensemblvep/vep/tests/main.nf.test @@ -31,7 +31,7 @@ nextflow_process { """ input[0] = Channel.of([ [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true), [] ]) input[1] = params.vep_genome @@ -40,7 +40,7 @@ nextflow_process { input[4] = ENSEMBLVEP_DOWNLOAD.out.cache.map{ meta, cache -> [cache] } input[5] = Channel.value([ [id:"fasta"], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) input[6] = [] """ @@ -76,7 +76,7 @@ nextflow_process { """ input[0] = Channel.of([ [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true), [] ]) input[1] = params.vep_genome @@ -85,7 +85,7 @@ nextflow_process { input[4] = ENSEMBLVEP_DOWNLOAD.out.cache.map{ meta, cache -> [cache] } input[5] = Channel.value([ [id:"fasta"], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) input[6] = [] """ From 60a7dbae179bcfa24c10294cc9a07423a239c19a Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Wed, 20 Mar 2024 09:32:19 +0100 Subject: [PATCH 06/19] Add README to modules build with Docker (#4935) * add Dockerfile * remove environment.yml * EC lint * add comment * remove Dockerfile * add README --------- Co-authored-by: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> --- modules/nf-core/deepvariant/README.md | 9 +++++++++ modules/nf-core/deepvariant/environment.yml | 5 ----- modules/nf-core/deepvariant/main.nf | 1 + modules/nf-core/gatk4/cnnscorevariants/README.md | 9 +++++++++ .../gatk4/determinegermlinecontigploidy/README.md | 9 +++++++++ modules/nf-core/gatk4/germlinecnvcaller/README.md | 9 +++++++++ .../nf-core/gatk4/postprocessgermlinecnvcalls/README.md | 9 +++++++++ 7 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 modules/nf-core/deepvariant/README.md delete mode 100644 modules/nf-core/deepvariant/environment.yml create mode 100644 modules/nf-core/gatk4/cnnscorevariants/README.md create mode 100644 modules/nf-core/gatk4/determinegermlinecontigploidy/README.md create mode 100644 modules/nf-core/gatk4/germlinecnvcaller/README.md create mode 100644 modules/nf-core/gatk4/postprocessgermlinecnvcalls/README.md diff --git a/modules/nf-core/deepvariant/README.md b/modules/nf-core/deepvariant/README.md new file mode 100644 index 00000000000..ca112a7d339 --- /dev/null +++ b/modules/nf-core/deepvariant/README.md @@ -0,0 +1,9 @@ +# Conda is not supported at the moment + +The [bioconda](https://bioconda.github.io/recipes/deepvariant/README.html) recipe is not fully working as expected + +Hence, we are using the docker container provided by the authors of the tool: + +- [google/deepvariant](https://hub.docker.com/r/google/deepvariant) + +This image is mirrored on the [nf-core quay.io](https://quay.io/repository/nf-core/deepvariant) for convenience. diff --git a/modules/nf-core/deepvariant/environment.yml b/modules/nf-core/deepvariant/environment.yml deleted file mode 100644 index 648a76dea69..00000000000 --- a/modules/nf-core/deepvariant/environment.yml +++ /dev/null @@ -1,5 +0,0 @@ -name: deepvariant -channels: - - conda-forge - - bioconda - - defaults diff --git a/modules/nf-core/deepvariant/main.nf b/modules/nf-core/deepvariant/main.nf index 2d5c480c4d4..507b6c1174b 100644 --- a/modules/nf-core/deepvariant/main.nf +++ b/modules/nf-core/deepvariant/main.nf @@ -2,6 +2,7 @@ process DEEPVARIANT { tag "$meta.id" label 'process_high' + //Conda is not supported at the moment container "nf-core/deepvariant:1.5.0" input: diff --git a/modules/nf-core/gatk4/cnnscorevariants/README.md b/modules/nf-core/gatk4/cnnscorevariants/README.md new file mode 100644 index 00000000000..c6a4545655a --- /dev/null +++ b/modules/nf-core/gatk4/cnnscorevariants/README.md @@ -0,0 +1,9 @@ +# Conda is not supported at the moment + +The [bioconda](https://bioconda.github.io/recipes/gatk4/README.html) recipe is not fully working as expected, cf [github issue](https://github.com/broadinstitute/gatk/issues/7811) + +Hence, we are using the docker container provided by the authors of the tool: + +- [broadinstitute/gatk](https://hub.docker.com/r/broadinstitute/gatk) + +This image is mirrored on the [nf-core quay.io](https://quay.io/repository/nf-core/gatk) for convenience. diff --git a/modules/nf-core/gatk4/determinegermlinecontigploidy/README.md b/modules/nf-core/gatk4/determinegermlinecontigploidy/README.md new file mode 100644 index 00000000000..c6a4545655a --- /dev/null +++ b/modules/nf-core/gatk4/determinegermlinecontigploidy/README.md @@ -0,0 +1,9 @@ +# Conda is not supported at the moment + +The [bioconda](https://bioconda.github.io/recipes/gatk4/README.html) recipe is not fully working as expected, cf [github issue](https://github.com/broadinstitute/gatk/issues/7811) + +Hence, we are using the docker container provided by the authors of the tool: + +- [broadinstitute/gatk](https://hub.docker.com/r/broadinstitute/gatk) + +This image is mirrored on the [nf-core quay.io](https://quay.io/repository/nf-core/gatk) for convenience. diff --git a/modules/nf-core/gatk4/germlinecnvcaller/README.md b/modules/nf-core/gatk4/germlinecnvcaller/README.md new file mode 100644 index 00000000000..c6a4545655a --- /dev/null +++ b/modules/nf-core/gatk4/germlinecnvcaller/README.md @@ -0,0 +1,9 @@ +# Conda is not supported at the moment + +The [bioconda](https://bioconda.github.io/recipes/gatk4/README.html) recipe is not fully working as expected, cf [github issue](https://github.com/broadinstitute/gatk/issues/7811) + +Hence, we are using the docker container provided by the authors of the tool: + +- [broadinstitute/gatk](https://hub.docker.com/r/broadinstitute/gatk) + +This image is mirrored on the [nf-core quay.io](https://quay.io/repository/nf-core/gatk) for convenience. diff --git a/modules/nf-core/gatk4/postprocessgermlinecnvcalls/README.md b/modules/nf-core/gatk4/postprocessgermlinecnvcalls/README.md new file mode 100644 index 00000000000..c6a4545655a --- /dev/null +++ b/modules/nf-core/gatk4/postprocessgermlinecnvcalls/README.md @@ -0,0 +1,9 @@ +# Conda is not supported at the moment + +The [bioconda](https://bioconda.github.io/recipes/gatk4/README.html) recipe is not fully working as expected, cf [github issue](https://github.com/broadinstitute/gatk/issues/7811) + +Hence, we are using the docker container provided by the authors of the tool: + +- [broadinstitute/gatk](https://hub.docker.com/r/broadinstitute/gatk) + +This image is mirrored on the [nf-core quay.io](https://quay.io/repository/nf-core/gatk) for convenience. From 3afb95b2e15fc4a2347470255a7ef654f650c8ec Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:21:38 +0100 Subject: [PATCH 07/19] add cram/index support to bwamem2 (#5248) * add cram/index support to bwamem2 * update tests --- modules/nf-core/bwamem2/mem/main.nf | 39 +++++++++++++++++-- modules/nf-core/bwamem2/mem/meta.yml | 27 +++++++++++++ .../nf-core/bwamem2/mem/tests/main.nf.test | 25 +++++++----- .../bwamem2/mem/tests/main.nf.test.snap | 30 +++++++------- subworkflows/nf-core/fastq_align_dna/main.nf | 2 +- 5 files changed, 93 insertions(+), 30 deletions(-) diff --git a/modules/nf-core/bwamem2/mem/main.nf b/modules/nf-core/bwamem2/mem/main.nf index 29f90778777..729428c4e07 100644 --- a/modules/nf-core/bwamem2/mem/main.nf +++ b/modules/nf-core/bwamem2/mem/main.nf @@ -10,11 +10,16 @@ process BWAMEM2_MEM { input: tuple val(meta), path(reads) tuple val(meta2), path(index) + tuple val(meta3), path(fasta) val sort_bam output: - tuple val(meta), path("*.bam"), emit: bam - path "versions.yml" , emit: versions + tuple val(meta), path("*.sam") , emit: sam , optional:true + tuple val(meta), path("*.bam") , emit: bam , optional:true + tuple val(meta), path("*.cram") , emit: cram, optional:true + tuple val(meta), path("*.crai") , emit: crai, optional:true + tuple val(meta), path("*.csi") , emit: csi , optional:true + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -24,6 +29,13 @@ process BWAMEM2_MEM { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def samtools_command = sort_bam ? 'sort' : 'view' + + def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ + def extension_matcher = (args2 =~ extension_pattern) + def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" + def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" + if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + """ INDEX=`find -L ./ -name "*.amb" | sed 's/\\.amb\$//'` @@ -33,7 +45,7 @@ process BWAMEM2_MEM { -t $task.cpus \\ \$INDEX \\ $reads \\ - | samtools $samtools_command $args2 -@ $task.cpus -o ${prefix}.bam - + | samtools $samtools_command $args2 -@ $task.cpus ${reference} -o ${prefix}.${extension} - cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -43,9 +55,28 @@ process BWAMEM2_MEM { """ stub: + + def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def samtools_command = sort_bam ? 'sort' : 'view' + def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ + def extension_matcher = (args2 =~ extension_pattern) + def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" + def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" + if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + + def create_index = "" + if (extension == "cram") { + create_index = "touch ${prefix}.crai" + } else if (extension == "bam") { + create_index = "touch ${prefix}.csi" + } + """ - touch ${prefix}.bam + touch ${prefix}.${extension} + ${create_index} + cat <<-END_VERSIONS > versions.yml "${task.process}": bwamem2: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') diff --git a/modules/nf-core/bwamem2/mem/meta.yml b/modules/nf-core/bwamem2/mem/meta.yml index 04891b26a97..931f712943d 100644 --- a/modules/nf-core/bwamem2/mem/meta.yml +++ b/modules/nf-core/bwamem2/mem/meta.yml @@ -37,6 +37,15 @@ input: type: file description: BWA genome index files pattern: "Directory containing BWA index *.{0132,amb,ann,bwt.2bit.64,pac}" + - meta3: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] + - fasta: + type: file + description: Reference genome in FASTA format + pattern: "*.{fa,fasta,fna}" - sort_bam: type: boolean description: use samtools sort (true) or samtools view (false) @@ -47,15 +56,33 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - sam: + type: file + description: Output SAM file containing read alignments + pattern: "*.{sam}" - bam: type: file description: Output BAM file containing read alignments pattern: "*.{bam}" + - cram: + type: file + description: Output CRAM file containing read alignments + pattern: "*.{cram}" + - crai: + type: file + description: Index file for CRAM file + pattern: "*.{crai}" + - csi: + type: file + description: Index file for BAM file + pattern: "*.{csi}" - versions: type: file description: File containing software versions pattern: "versions.yml" authors: - "@maxulysse" + - "@matthdsm" maintainers: - "@maxulysse" + - "@matthdsm" diff --git a/modules/nf-core/bwamem2/mem/tests/main.nf.test b/modules/nf-core/bwamem2/mem/tests/main.nf.test index 365a0c433c1..5e67f70b6a2 100644 --- a/modules/nf-core/bwamem2/mem/tests/main.nf.test +++ b/modules/nf-core/bwamem2/mem/tests/main.nf.test @@ -10,7 +10,7 @@ nextflow_process { tag "bwamem2/mem" tag "bwamem2/index" - test("sarscov2 - fastq, index, false") { + test("sarscov2 - fastq, index, fasta, false") { setup { run("BWAMEM2_INDEX") { @@ -34,7 +34,8 @@ nextflow_process { [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] ]) input[1] = BWAMEM2_INDEX.out.index - input[2] = false + input[2] = Channel.of([[:], [file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)]]) + input[3] = false """ } } @@ -51,7 +52,7 @@ nextflow_process { } - test("sarscov2 - fastq, index, true") { + test("sarscov2 - fastq, index, fasta, true") { setup { run("BWAMEM2_INDEX") { @@ -75,7 +76,8 @@ nextflow_process { [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] ]) input[1] = BWAMEM2_INDEX.out.index - input[2] = true + input[2] = Channel.of([[:], [file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)]]) + input[3] = true """ } } @@ -92,7 +94,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, false") { + test("sarscov2 - [fastq1, fastq2], index, fasta, false") { setup { run("BWAMEM2_INDEX") { @@ -119,7 +121,8 @@ nextflow_process { ] ]) input[1] = BWAMEM2_INDEX.out.index - input[2] = false + input[2] = Channel.of([[:], [file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)]]) + input[3] = false """ } } @@ -136,7 +139,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, true") { + test("sarscov2 - [fastq1, fastq2], index, fasta, true") { setup { run("BWAMEM2_INDEX") { @@ -163,7 +166,8 @@ nextflow_process { ] ]) input[1] = BWAMEM2_INDEX.out.index - input[2] = true + input[2] = Channel.of([[:], [file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)]]) + input[3] = true """ } } @@ -180,7 +184,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, true - stub") { + test("sarscov2 - [fastq1, fastq2], index, fasta, true - stub") { options "-stub" @@ -209,7 +213,8 @@ nextflow_process { ] ]) input[1] = BWAMEM2_INDEX.out.index - input[2] = true + input[2] = Channel.of([[:], [file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)]]) + input[3] = true """ } } diff --git a/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap b/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap index 84be71c60c7..9fb1e69d07d 100644 --- a/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap +++ b/modules/nf-core/bwamem2/mem/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sarscov2 - [fastq1, fastq2], index, true": { + "sarscov2 - [fastq1, fastq2], index, fasta, false": { "content": [ "test.bam", [ @@ -8,11 +8,11 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T13:30:22.691288603" + "timestamp": "2024-03-19T13:13:18.890289958" }, - "sarscov2 - [fastq1, fastq2], index, false": { + "sarscov2 - [fastq1, fastq2], index, fasta, true - stub": { "content": [ "test.bam", [ @@ -21,11 +21,11 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T13:30:11.276168706" + "timestamp": "2024-03-19T13:45:51.821633029" }, - "sarscov2 - [fastq1, fastq2], index, true - stub": { + "sarscov2 - [fastq1, fastq2], index, fasta, true": { "content": [ "test.bam", [ @@ -34,11 +34,11 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T13:30:32.07431961" + "timestamp": "2024-03-19T13:13:36.458291078" }, - "sarscov2 - fastq, index, false": { + "sarscov2 - fastq, index, fasta, false": { "content": [ "test.bam", [ @@ -47,11 +47,11 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T13:29:48.586760544" + "timestamp": "2024-03-19T13:12:44.084654507" }, - "sarscov2 - fastq, index, true": { + "sarscov2 - fastq, index, fasta, true": { "content": [ "test.bam", [ @@ -60,8 +60,8 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T13:29:59.846686393" + "timestamp": "2024-03-19T13:13:01.763341681" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_dna/main.nf b/subworkflows/nf-core/fastq_align_dna/main.nf index f907b16e47a..b710a4a6403 100644 --- a/subworkflows/nf-core/fastq_align_dna/main.nf +++ b/subworkflows/nf-core/fastq_align_dna/main.nf @@ -42,7 +42,7 @@ workflow FASTQ_ALIGN_DNA { ch_versions = ch_versions.mix(BWAMEM1_MEM.out.versions) break case 'bwamem2': - BWAMEM2_MEM (ch_reads, ch_aligner_index, sort) // If aligner is bwa-mem2 + BWAMEM2_MEM (ch_reads, ch_aligner_index, ch_fasta, sort) // If aligner is bwa-mem2 ch_bam = ch_bam.mix(BWAMEM2_MEM.out.bam) ch_versions = ch_versions.mix(BWAMEM2_MEM.out.versions) break From 653218e79ffa76fde20319e9062f8b8da5cf7555 Mon Sep 17 00:00:00 2001 From: Joon Klaps <61584065+Joon-Klaps@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:22:20 +0100 Subject: [PATCH 08/19] 4557 new module kaijumergeoutputs + stub Kraken2/Kraken2 (#5249) * init kaiju-mergedOutputs * init kaiju_mergeoutputs * Update file paths and add test case for stub in kraken2_kraken2 * adressing comments reviewer * Fix variable assignment in db.config and nodb.config * Add docker.fixOwnership to process block in db.config and nodb.config * Update process configuration to fix ownership issue * typo * Remove docker.fixOwnership from db.config --- .../kaiju/mergeoutputs/environment.yml | 9 ++ modules/nf-core/kaiju/mergeoutputs/main.nf | 53 +++++++ modules/nf-core/kaiju/mergeoutputs/meta.yml | 53 +++++++ .../kaiju/mergeoutputs/tests/db.config | 5 + .../kaiju/mergeoutputs/tests/main.nf.test | 133 ++++++++++++++++++ .../mergeoutputs/tests/main.nf.test.snap | 107 ++++++++++++++ .../kaiju/mergeoutputs/tests/nodb.config | 14 ++ .../nf-core/kaiju/mergeoutputs/tests/tags.yml | 2 + modules/nf-core/kraken2/kraken2/main.nf | 27 ++++ 9 files changed, 403 insertions(+) create mode 100644 modules/nf-core/kaiju/mergeoutputs/environment.yml create mode 100644 modules/nf-core/kaiju/mergeoutputs/main.nf create mode 100644 modules/nf-core/kaiju/mergeoutputs/meta.yml create mode 100644 modules/nf-core/kaiju/mergeoutputs/tests/db.config create mode 100644 modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test create mode 100644 modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test.snap create mode 100644 modules/nf-core/kaiju/mergeoutputs/tests/nodb.config create mode 100644 modules/nf-core/kaiju/mergeoutputs/tests/tags.yml diff --git a/modules/nf-core/kaiju/mergeoutputs/environment.yml b/modules/nf-core/kaiju/mergeoutputs/environment.yml new file mode 100644 index 00000000000..c42f573c131 --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "kaiju_mergeoutputs" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::kaiju=1.10.0" diff --git a/modules/nf-core/kaiju/mergeoutputs/main.nf b/modules/nf-core/kaiju/mergeoutputs/main.nf new file mode 100644 index 00000000000..a3c5296e859 --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/main.nf @@ -0,0 +1,53 @@ +process KAIJU_MERGEOUTPUTS { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/kaiju:1.10.0--h43eeafb_0': + 'biocontainers/kaiju:1.10.0--h43eeafb_0' }" + + input: + tuple val(meta), path(kaiju), path(kraken) + path (db) + + output: + tuple val(meta), path("*.tsv"), emit: merged + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def dbnodes = db ? '-t <(find -L ${db} -name "*nodes.dmp")' : '' + script: + if ("$kaiju" == "${prefix}.tsv") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$kraken" == "${prefix}.tsv") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + """ + kaiju-mergeOutputs \\ + -i <(sort -k2,2 ${kaiju}) \\ + -j <(sort -k2,2 ${kraken}) \\ + -o ${prefix}.tsv \\ + $dbnodes \\ + $args + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + kaiju: \$(echo \$( kaiju -h 2>&1 | sed -n 1p | sed 's/^.*Kaiju //' )) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.tsv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + kaiju: \$(echo \$( kaiju -h 2>&1 | sed -n 1p | sed 's/^.*Kaiju //' )) + END_VERSIONS + """ +} diff --git a/modules/nf-core/kaiju/mergeoutputs/meta.yml b/modules/nf-core/kaiju/mergeoutputs/meta.yml new file mode 100644 index 00000000000..4ec5000344c --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/meta.yml @@ -0,0 +1,53 @@ +name: kaiju_mergeoutputs +description: Merge two tab-separated output files of Kaiju and Kraken in the column format +keywords: + - classify + - metagenomics + - fastq + - taxonomic profiling +tools: + - kaiju: + description: Fast and sensitive taxonomic classification for metagenomics + homepage: https://kaiju.binf.ku.dk/ + documentation: https://github.com/bioinformatics-centre/kaiju/blob/master/README.md + tool_dev_url: https://github.com/bioinformatics-centre/kaiju + doi: "10.1038/ncomms11257" + licence: ["GNU GPL v3"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - kaiju: + type: file + description: | + Results with taxonomic classification of each read from Kaiju + - kraken: + type: file + description: | + Results with taxonomic classification of each read from Kraken + pattern: "*.{tsv}" + - db: + type: directory + description: | + List containing the database and nodes files for Kaiju + e.g. [ 'database.fmi', 'nodes.dmp' ] +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - merged: + type: file + description: Results with merged taxonomic classification of each read based on the given strategy '1', '2', 'lca' [default] or 'lowest' + pattern: "*.tsv" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@Joon-Klaps" +maintainers: + - "@Joon-Klaps" diff --git a/modules/nf-core/kaiju/mergeoutputs/tests/db.config b/modules/nf-core/kaiju/mergeoutputs/tests/db.config new file mode 100644 index 00000000000..257dba4704b --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/tests/db.config @@ -0,0 +1,5 @@ +process { + withName: KAIJU_MERGEOUTPUTS{ + ext.prefix = {"${meta.id}_merged"} + } +} diff --git a/modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test b/modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test new file mode 100644 index 00000000000..5c2d5b3159d --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test @@ -0,0 +1,133 @@ +nextflow_process { + + name "Test Process KAIJU_MERGEOUTPUTS" + script "../main.nf" + process "KAIJU_MERGEOUTPUTS" + + tag "modules" + tag "modules_nfcore" + tag "kaiju" + tag "untar" + tag "kaiju/kaiju" + tag "kraken2/kraken2" + tag "kaiju/mergeoutputs" + + setup { + run("UNTAR",alias: "UNTAR_KRAKEN2" ) { + config "./nodb.config" + script "../../../untar/main.nf" + process { + """ + input[0] = Channel.of([ + [], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) + ]) + """ + } + } + run("KRAKEN2_KRAKEN2") { + config "./nodb.config" + script "../../../kraken2/kraken2/main.nf" + process { + """ + input[0] = [ + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ] + input[1] = UNTAR_KRAKEN2.out.untar.map{ it[1] } + input[2] = false + input[3] = true + """ + } + } + run("UNTAR",alias: "UNTAR_KAIJU" ) { + config "./nodb.config" + script "../../../untar/main.nf" + process { + """ + input[0] = Channel.of([ + [], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kaiju.tar.gz', checkIfExists: true) + ]) + """ + } + } + run("KAIJU_KAIJU") { + config "./nodb.config" + script "../../../kaiju/kaiju/main.nf" + process { + """ + input[0] = [ + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ] + input[1] = UNTAR_KAIJU.out.untar.map{ it[1] } + """ + } + } + } + + test("sarscov2 - kraken2.kaiju.classified - noDB ") { + config "./nodb.config" + + when { + process { + """ + input[0] = KAIJU_KAIJU.out.results.join(KRAKEN2_KRAKEN2.out.classified_reads_assignment) + input[1] = [ ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("sarscov2 - kraken2.kaiju.classified - kaiju.DB ") { + config "./db.config" + when { + process { + """ + input[0] = KAIJU_KAIJU.out.results.join(KRAKEN2_KRAKEN2.out.classified_reads_assignment) + input[1] = UNTAR_KAIJU.out.untar.map{ it[1] } + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("sarscov2 - kraken2.kaiju.classified - noDB - stub ") { + config "./nodb.config" + options "-stub" + + when { + process { + """ + input[0] = KAIJU_KAIJU.out.results.join(KRAKEN2_KRAKEN2.out.classified_reads_assignment) + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test.snap b/modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test.snap new file mode 100644 index 00000000000..7ee12505930 --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/tests/main.nf.test.snap @@ -0,0 +1,107 @@ +{ + "sarscov2 - kraken2.kaiju.classified - noDB - stub ": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test_merged.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,fe28eae98973762f4fb19fc562a56b9f" + ], + "merged": [ + [ + { + "id": "test", + "single_end": true + }, + "test_merged.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,fe28eae98973762f4fb19fc562a56b9f" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T13:48:25.353007932" + }, + "sarscov2 - kraken2.kaiju.classified - noDB ": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test_merged.tsv:md5,64fef9779d5003775c327bb534403859" + ] + ], + "1": [ + "versions.yml:md5,fe28eae98973762f4fb19fc562a56b9f" + ], + "merged": [ + [ + { + "id": "test", + "single_end": true + }, + "test_merged.tsv:md5,64fef9779d5003775c327bb534403859" + ] + ], + "versions": [ + "versions.yml:md5,fe28eae98973762f4fb19fc562a56b9f" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T13:47:53.326864948" + }, + "sarscov2 - kraken2.kaiju.classified - kaiju.DB ": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test_merged.tsv:md5,64fef9779d5003775c327bb534403859" + ] + ], + "1": [ + "versions.yml:md5,fe28eae98973762f4fb19fc562a56b9f" + ], + "merged": [ + [ + { + "id": "test", + "single_end": true + }, + "test_merged.tsv:md5,64fef9779d5003775c327bb534403859" + ] + ], + "versions": [ + "versions.yml:md5,fe28eae98973762f4fb19fc562a56b9f" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T13:48:11.274433603" + } +} \ No newline at end of file diff --git a/modules/nf-core/kaiju/mergeoutputs/tests/nodb.config b/modules/nf-core/kaiju/mergeoutputs/tests/nodb.config new file mode 100644 index 00000000000..e6f6332b3c5 --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/tests/nodb.config @@ -0,0 +1,14 @@ +process { + withName: UNTAR_KAIJU{ + ext.args2 = {"--no-same-owner"} + } + + withName: UNTAR_KRAKEN2{ + ext.args2 = {"--no-same-owner"} + } + + withName: KAIJU_MERGEOUTPUTS{ + ext.prefix = {"${meta.id}_merged"} + ext.args= "-c 1" + } +} diff --git a/modules/nf-core/kaiju/mergeoutputs/tests/tags.yml b/modules/nf-core/kaiju/mergeoutputs/tests/tags.yml new file mode 100644 index 00000000000..8b1f3c8634d --- /dev/null +++ b/modules/nf-core/kaiju/mergeoutputs/tests/tags.yml @@ -0,0 +1,2 @@ +kaiju/mergeoutputs: + - "modules/nf-core/kaiju/mergeoutputs/**" diff --git a/modules/nf-core/kraken2/kraken2/main.nf b/modules/nf-core/kraken2/kraken2/main.nf index cdf02d0f145..92cd9c34f7a 100644 --- a/modules/nf-core/kraken2/kraken2/main.nf +++ b/modules/nf-core/kraken2/kraken2/main.nf @@ -55,4 +55,31 @@ process KRAKEN2_KRAKEN2 { pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) END_VERSIONS """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def paired = meta.single_end ? "" : "--paired" + def classified = meta.single_end ? "${prefix}.classified.fastq.gz" : "${prefix}.classified_1.fastq.gz ${prefix}.classified_2.fastq.gz" + def unclassified = meta.single_end ? "${prefix}.unclassified.fastq.gz" : "${prefix}.unclassified_1.fastq.gz ${prefix}.unclassified_2.fastq.gz" + def readclassification_option = save_reads_assignment ? "--output ${prefix}.kraken2.classifiedreads.txt" : "--output /dev/null" + def compress_reads_command = save_output_fastqs ? "pigz -p $task.cpus *.fastq" : "" + + """ + touch ${prefix}.kraken2.report.txt + if [ "$save_output_fastqs" == "true" ]; then + touch $classified + touch $unclassified + fi + if [ "$save_reads_assignment" == "true" ]; then + touch ${prefix}.kraken2.classifiedreads.txt + fi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + kraken2: \$(echo \$(kraken2 --version 2>&1) | sed 's/^.*Kraken version //; s/ .*\$//') + pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) + END_VERSIONS + """ + } From 89cf292ab826db67279cf11339c5afc53a9e419d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luc=C3=ADa=20Pe=C3=B1a-P=C3=A9rez?= Date: Wed, 20 Mar 2024 09:26:07 +0100 Subject: [PATCH 09/19] gatk4_asereadcounter add updated meta and nf-tests (#5164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat updated meta * feat add nf-tests * Update modules/nf-core/gatk4/asereadcounter/meta.yml Co-authored-by: Matthias Hörtenhuber * Update modules/nf-core/gatk4/asereadcounter/meta.yml Co-authored-by: Matthias Hörtenhuber * applyed reviewers suggestions * modified pytest_modules.yml --------- Co-authored-by: lucia.pena.perez@scilifelab.se Co-authored-by: Matthias Hörtenhuber --- modules/nf-core/gatk4/asereadcounter/main.nf | 8 +-- .../nf-core/gatk4/asereadcounter/main.nf.test | 54 +++++++++++++++++++ .../gatk4/asereadcounter/main.nf.test.snap | 35 ++++++++++++ modules/nf-core/gatk4/asereadcounter/meta.yml | 28 ++++++++++ tests/config/pytest_modules.yml | 6 +-- .../nf-core/gatk4/asereadcounter/main.nf | 23 -------- .../gatk4/asereadcounter/nextflow.config | 5 -- .../nf-core/gatk4/asereadcounter/test.yml | 17 ------ 8 files changed, 124 insertions(+), 52 deletions(-) create mode 100644 modules/nf-core/gatk4/asereadcounter/main.nf.test create mode 100644 modules/nf-core/gatk4/asereadcounter/main.nf.test.snap delete mode 100644 tests/modules/nf-core/gatk4/asereadcounter/main.nf delete mode 100644 tests/modules/nf-core/gatk4/asereadcounter/nextflow.config delete mode 100644 tests/modules/nf-core/gatk4/asereadcounter/test.yml diff --git a/modules/nf-core/gatk4/asereadcounter/main.nf b/modules/nf-core/gatk4/asereadcounter/main.nf index 4a151d6c6e7..9b5b62a2061 100644 --- a/modules/nf-core/gatk4/asereadcounter/main.nf +++ b/modules/nf-core/gatk4/asereadcounter/main.nf @@ -9,10 +9,10 @@ process GATK4_ASEREADCOUNTER { input: tuple val(meta), path(input), path(input_index) - tuple val(meta), path(vcf), path(tbi) - path fasta - path fai - path dict + tuple val(meta2), path(vcf), path(tbi) + tuple val(meta3), path(fasta) + tuple val(meta4), path(fai) + tuple val(meta5), path(dict) path intervals output: diff --git a/modules/nf-core/gatk4/asereadcounter/main.nf.test b/modules/nf-core/gatk4/asereadcounter/main.nf.test new file mode 100644 index 00000000000..e532d38d9af --- /dev/null +++ b/modules/nf-core/gatk4/asereadcounter/main.nf.test @@ -0,0 +1,54 @@ +nextflow_process { + + name "Test Process GATK4_ASEREADCOUNTER" + tag "modules_nfcore" + tag "modules" + tag "gatk4" + tag "gatk4/asereadcounter" + script "modules/nf-core/gatk4/asereadcounter/main.nf" + process "GATK4_ASEREADCOUNTER" + + test("Should run without failures") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path +'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path +'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true) + ] + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path +'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path +'genomics/sarscov2/illumina/vcf/test.vcf.gz.tbi', checkIfExists: true) + ] + input[2] = [ + [ id:'reference' ], // meta map + file(params.modules_testdata_base_path +'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + input[3] = [ + [ id:'reference' ], // meta map + file(params.modules_testdata_base_path +'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) + ] + input[4] = [ + [ id:'reference' ], // meta map + file(params.modules_testdata_base_path +'genomics/sarscov2/genome/genome.dict', checkIfExists: true) + ] + input[5] = [ + file(params.modules_testdata_base_path +'genomics/sarscov2/genome/picard/targets.interval_list', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + +} diff --git a/modules/nf-core/gatk4/asereadcounter/main.nf.test.snap b/modules/nf-core/gatk4/asereadcounter/main.nf.test.snap new file mode 100644 index 00000000000..cb2740864b2 --- /dev/null +++ b/modules/nf-core/gatk4/asereadcounter/main.nf.test.snap @@ -0,0 +1,35 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test_ase.csv:md5,4fbcaf6b6a390889c56e105c41470b7b" + ] + ], + "1": [ + "versions.yml:md5,d567b20620f5adf0829cc4b4adb9f6ff" + ], + "csv": [ + [ + { + "id": "test" + }, + "test_ase.csv:md5,4fbcaf6b6a390889c56e105c41470b7b" + ] + ], + "versions": [ + "versions.yml:md5,d567b20620f5adf0829cc4b4adb9f6ff" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T10:03:42.158801" + } +} \ No newline at end of file diff --git a/modules/nf-core/gatk4/asereadcounter/meta.yml b/modules/nf-core/gatk4/asereadcounter/meta.yml index 2282322deee..7367dec7eca 100644 --- a/modules/nf-core/gatk4/asereadcounter/meta.yml +++ b/modules/nf-core/gatk4/asereadcounter/meta.yml @@ -25,18 +25,46 @@ input: type: file description: BAM/CRAM/SAM file pattern: "*.{bam,cram,sam}" + - input_index: + type: file + description: index file for input file + pattern: "*.{bai,crai}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. `[ id:'reference' ]` - vcf: type: file description: VCF file pattern: "*.{vcf.gz}" + - tbi: + type: file + description: index file for VCF file + pattern: "*.{vcf.gz.tbi}" + - meta3: + type: map + description: | + Groovy Map containing reference information + e.g. `[ id:'reference' ]` - fasta: type: file description: fasta file pattern: "*.{fasta,fa}" + - meta4: + type: map + description: | + Groovy Map containing reference information + e.g. `[ id:'reference' ]` - fai: type: file description: fasta index file pattern: "*.{fai}" + - meta5: + type: map + description: | + Groovy Map containing reference information + e.g. `[ id:'reference' ]` - dict: type: file description: dictionary file diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index d9733192abe..d72dc4ddf8a 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -697,9 +697,9 @@ gatk/unifiedgenotyper: gatk4/applyvqsr: - modules/nf-core/gatk4/applyvqsr/** - tests/modules/nf-core/gatk4/applyvqsr/** -gatk4/asereadcounter: - - modules/nf-core/gatk4/asereadcounter/** - - tests/modules/nf-core/gatk4/asereadcounter/** +gatk4/bedtointervallist: + - modules/nf-core/gatk4/bedtointervallist/** + - tests/modules/nf-core/gatk4/bedtointervallist/** gatk4/calculatecontamination: - modules/nf-core/gatk4/calculatecontamination/** - tests/modules/nf-core/gatk4/calculatecontamination/** diff --git a/tests/modules/nf-core/gatk4/asereadcounter/main.nf b/tests/modules/nf-core/gatk4/asereadcounter/main.nf deleted file mode 100644 index 0af3d0a2bb1..00000000000 --- a/tests/modules/nf-core/gatk4/asereadcounter/main.nf +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { GATK4_ASEREADCOUNTER } from '../../../../../modules/nf-core/gatk4/asereadcounter/main.nf' - -workflow test_gatk4_asereadcounter { - - input = [ [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) - ] - vcf = [ [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true) - ] - fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - fai = file(params.test_data['sarscov2']['genome']['genome_fasta_fai'], checkIfExists: true) - dict = file(params.test_data['sarscov2']['genome']['genome_dict'], checkIfExists: true) - intervals = file(params.test_data['sarscov2']['genome']['targets_interval_list'], checkIfExists: true) - - GATK4_ASEREADCOUNTER ( input, vcf, fasta, fai, dict, intervals ) -} diff --git a/tests/modules/nf-core/gatk4/asereadcounter/nextflow.config b/tests/modules/nf-core/gatk4/asereadcounter/nextflow.config deleted file mode 100644 index 50f50a7a357..00000000000 --- a/tests/modules/nf-core/gatk4/asereadcounter/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} \ No newline at end of file diff --git a/tests/modules/nf-core/gatk4/asereadcounter/test.yml b/tests/modules/nf-core/gatk4/asereadcounter/test.yml deleted file mode 100644 index 98ee2ffb50a..00000000000 --- a/tests/modules/nf-core/gatk4/asereadcounter/test.yml +++ /dev/null @@ -1,17 +0,0 @@ -- name: gatk4 asereadcounter test_gatk4_asereadcounter - command: nextflow run ./tests/modules/nf-core/gatk4/asereadcounter -entry test_gatk4_asereadcounter -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/gatk4/asereadcounter/nextflow.config - tags: - - gatk4/asereadcounter - - gatk4 - files: - - path: output/gatk4/test_ase.csv - md5sum: 4fbcaf6b6a390889c56e105c41470b7b - - path: output/gatk4/versions.yml -- name: gatk4 asereadcounter test_gatk4_asereadcounter_stubs - command: nextflow run ./tests/modules/nf-core/gatk4/asereadcounter -entry test_gatk4_asereadcounter -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/gatk4/asereadcounter/nextflow.config -stub-run - tags: - - gatk4/asereadcounter - - gatk4 - files: - - path: output/gatk4/test_ase.csv - - path: output/gatk4/versions.yml From 8487a4441d6e49f443fc29006746eb12382f2e19 Mon Sep 17 00:00:00 2001 From: John Orgera <65687576+johnoooh@users.noreply.github.com> Date: Wed, 20 Mar 2024 04:39:05 -0400 Subject: [PATCH 10/19] Bamstats (#4474) * initial commit * commit to test * added :: * changed md5 * back to : * added comments and md5 * removed . * added bamstats to a line * spaces after colons * colon issues * sed tests * sed changes * sed fix * ' fix * sed fix * uising back * fix removed * sed fixed hopefully * md5 fix * removed todo * documentation added to meta.yml * removed todos * removed todo * fixed whitespace * fix licence * whitespace * whitespace again * para change * para change * md5 change * Update modules/nf-core/bamstats/generalstats/main.nf Co-authored-by: Gisela Gabernet * todo removed * nf-test conversion * Fix bamstats module test.yml * Migrate bamstats tests to nf-test * Update bamstats/generalstats module and tests --------- Co-authored-by: Gisela Gabernet Co-authored-by: Emilio Palumbo --- .../bamstats/generalstats/environment.yml | 9 +++ modules/nf-core/bamstats/generalstats/main.nf | 51 ++++++++++++++++ .../nf-core/bamstats/generalstats/meta.yml | 46 ++++++++++++++ .../bamstats/generalstats/tests/main.nf.test | 60 +++++++++++++++++++ .../generalstats/tests/main.nf.test.snap | 26 ++++++++ .../bamstats/generalstats/tests/tags.yml | 2 + 6 files changed, 194 insertions(+) create mode 100644 modules/nf-core/bamstats/generalstats/environment.yml create mode 100644 modules/nf-core/bamstats/generalstats/main.nf create mode 100644 modules/nf-core/bamstats/generalstats/meta.yml create mode 100644 modules/nf-core/bamstats/generalstats/tests/main.nf.test create mode 100644 modules/nf-core/bamstats/generalstats/tests/main.nf.test.snap create mode 100644 modules/nf-core/bamstats/generalstats/tests/tags.yml diff --git a/modules/nf-core/bamstats/generalstats/environment.yml b/modules/nf-core/bamstats/generalstats/environment.yml new file mode 100644 index 00000000000..71d07d16ef1 --- /dev/null +++ b/modules/nf-core/bamstats/generalstats/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "bamstats_generalstats" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::bamstats=0.3.5" diff --git a/modules/nf-core/bamstats/generalstats/main.nf b/modules/nf-core/bamstats/generalstats/main.nf new file mode 100644 index 00000000000..4bae58d6ba1 --- /dev/null +++ b/modules/nf-core/bamstats/generalstats/main.nf @@ -0,0 +1,51 @@ +process BAMSTATS_GENERALSTATS { + tag "$meta.id" + label 'process_single' + conda "bioconda::bamstats=0.3.5" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bamstats:0.3.5--he881be0_0': + 'biocontainers/bamstats:0.3.5--he881be0_0' }" + + input: + tuple val(meta), path(bam) + + output: + tuple val(meta), path("*.json"), emit: json + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // Several ARGS are available. + // -a is a helpfu; one where you can add a BED file + // -u, --uniq outputs genomic coverage statistics for uniqely mapped reads + """ + bamstats \\ + -i $bam \\ + $args \\ + -c $task.cpus \\ + -o ${prefix}.json + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bamstats: \$(echo \$(bamstats --version 2>&1) | sed 's/^.*bamstats == version://; s/Using.*\$//' | sed 's/built.*//' ) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + touch ${prefix}.json + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bamstats: \$(echo \$(bamstats --version 2>&1) | sed 's/^.*bamstats == version://; s/Using.*\$//' | sed 's/built.*//' ) + END_VERSIONS + """ +} + diff --git a/modules/nf-core/bamstats/generalstats/meta.yml b/modules/nf-core/bamstats/generalstats/meta.yml new file mode 100644 index 00000000000..a0921f4a3e3 --- /dev/null +++ b/modules/nf-core/bamstats/generalstats/meta.yml @@ -0,0 +1,46 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +name: "bamstats_generalstats" +description: write your description here +keywords: + - bam + - statistics + - genomics +tools: + - "bamstats": + description: "A command line tool to compute mapping statistics from a BAM file" + homepage: "https://github.com/guigolab/bamstats/" + documentation: "https://github.com/guigolab/bamstats/" + tool_dev_url: "https://github.com/guigolab" + licence: ["BSD-3-clause"] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'test', single_end:false ]` + + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'test', single_end:false ]` + + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - json: + type: file + description: json containing bam statistics + pattern: "*.json" + +authors: + - "@johnoooh" diff --git a/modules/nf-core/bamstats/generalstats/tests/main.nf.test b/modules/nf-core/bamstats/generalstats/tests/main.nf.test new file mode 100644 index 00000000000..8a5a18b1104 --- /dev/null +++ b/modules/nf-core/bamstats/generalstats/tests/main.nf.test @@ -0,0 +1,60 @@ +nextflow_process { + + name "Test Process BAMSTATS_GENERALSTATS" + script "../main.nf" + process "BAMSTATS_GENERALSTATS" + + tag "modules" + tag "modules_nfcore" + tag "bamstats" + tag "bamstats/generalstats" + + test("sarscov2 - bam") { + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.json.get(0)[1]).exists() }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + + } + + test("sarscov2 - bam - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.json.get(0)[1]).exists() }, + { assert snapshot(process.out.versions).match("versions_stub") } + ) + } + + } + +} diff --git a/modules/nf-core/bamstats/generalstats/tests/main.nf.test.snap b/modules/nf-core/bamstats/generalstats/tests/main.nf.test.snap new file mode 100644 index 00000000000..c2244d1bcba --- /dev/null +++ b/modules/nf-core/bamstats/generalstats/tests/main.nf.test.snap @@ -0,0 +1,26 @@ +{ + "versions_stub": { + "content": [ + [ + "versions.yml:md5,6d858a3f76d1852b879f15436882c3cf" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T16:52:19.198122" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,6d858a3f76d1852b879f15436882c3cf" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T16:52:07.413799" + } +} \ No newline at end of file diff --git a/modules/nf-core/bamstats/generalstats/tests/tags.yml b/modules/nf-core/bamstats/generalstats/tests/tags.yml new file mode 100644 index 00000000000..d5bb0d9d1c7 --- /dev/null +++ b/modules/nf-core/bamstats/generalstats/tests/tags.yml @@ -0,0 +1,2 @@ +bamstats/generalstats: + - "modules/nf-core/bamstats/generalstats/**" From 61f2ea506bd87ef436b0086f91a07abc6035fcd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20B=C3=A4uerle?= <45968370+famosab@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:25:09 +0100 Subject: [PATCH 11/19] Add wittyer as module (#5171) * feat: add wittyer as module * fix: adjust output and version * fix: copy output and script from local * feat: fill in in meta file * fix: remove config file input * fix: run prettier * feat: explain how to create docker image * fix: README for Docker image * fix: modify environment file * fix: stub and file extensions * fix: remove custom extensions * feat: add test file - failing * fix: correct input for tests * fix: licence * fix: container name * fix: finally correct container * fix syntax errors * normal test works, stub fails * fix expected outputs * fix: mention making image public * fix add stageAs, use dotnet * run prettier * [automated] Fix linting with Prettier * add comment for run * fix trailing whitespace * add dependencies to fix linting * fix harmonize inputs in meta and main * fix: run prettier * [automated] Fix linting with Prettier * fix index names * fix version to stub_version in stub test Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> * extend tests to capture as much as possible * test with bed file * fix: remove stageAs * fix tests for stageAs removal * remove '' in meta file * exclude conda test for wittyer --------- Co-authored-by: nf-core-bot Co-authored-by: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> --- .github/workflows/test.yml | 2 + modules/nf-core/wittyer/README.md | 28 ++ modules/nf-core/wittyer/environment.yml | 7 + modules/nf-core/wittyer/main.nf | 69 ++++ modules/nf-core/wittyer/meta.yml | 66 ++++ modules/nf-core/wittyer/tests/main.nf.test | 100 +++++ .../nf-core/wittyer/tests/main.nf.test.snap | 347 ++++++++++++++++++ modules/nf-core/wittyer/tests/tags.yml | 2 + 8 files changed, 621 insertions(+) create mode 100644 modules/nf-core/wittyer/README.md create mode 100644 modules/nf-core/wittyer/environment.yml create mode 100644 modules/nf-core/wittyer/main.nf create mode 100644 modules/nf-core/wittyer/meta.yml create mode 100644 modules/nf-core/wittyer/tests/main.nf.test create mode 100644 modules/nf-core/wittyer/tests/main.nf.test.snap create mode 100644 modules/nf-core/wittyer/tests/tags.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd2dbcf793e..544b2cf26b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -585,6 +585,8 @@ jobs: tags: subworkflows/vcf_annotate_ensemblvep - profile: singularity tags: bases2fastq + - profile: conda + tags: wittyer env: NXF_ANSI_LOG: false SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} diff --git a/modules/nf-core/wittyer/README.md b/modules/nf-core/wittyer/README.md new file mode 100644 index 00000000000..d2ba0e09b41 --- /dev/null +++ b/modules/nf-core/wittyer/README.md @@ -0,0 +1,28 @@ +# Updating the docker container and making a new module release + +witty.er is a commercial tool from Illumina. The container provided for the witty.er nf-core module is not provided nor supported by Illumina. Updating the witty.er versions in the container and pushing the update to Dockerhub needs to be done manually. + +1. Navigate to the witty.er github repository. - [witty.er](https://github.com/Illumina/witty.er) +2. Download the latest release. + ```bash + wget https://github.com/Illumina/witty.er/archive/refs/tags/.tar.gz + ``` +3. Uncompress the released package. + ```bash + tar -xvf .tar.gz + ``` +4. Change to the uncompressed directory. +5. Build docker image using provided Dockerfile. + + ```bash + docker build -t wittyer: --platform linux/amd64 . + ``` + +6. Access rights are needed to push the container to the Dockerhub nfcore organization, please ask a core team member to do so. + + ```bash + docker tag wittyer: quay.io/nf-core/wittyer: + docker push quay.io/nf-core/wittyer: + ``` + +7. Make the image public. diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml new file mode 100644 index 00000000000..f8378df2479 --- /dev/null +++ b/modules/nf-core/wittyer/environment.yml @@ -0,0 +1,7 @@ +name: wittyer +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - tabix diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf new file mode 100644 index 00000000000..c2b943f393b --- /dev/null +++ b/modules/nf-core/wittyer/main.nf @@ -0,0 +1,69 @@ +process WITTYER { + tag "$meta.id" + label 'process_single' + + container "nf-core/wittyer:0.3.3.0" + + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "WITTYER module does not support Conda. Please use Docker / Singularity / Podman instead." + } + + input: + tuple val(meta), path(query_vcf), path(query_vcf_index), path(truth_vcf), path(truth_vcf_index), path(bed) + + output: + tuple val(meta), path("*.json") , emit: report + tuple val(meta), path("*.vcf.gz") , emit: bench_vcf + tuple val(meta), path("*.vcf.gz.tbi") , emit: bench_vcf_tbi + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def regions = bed ? "--includeBed=$bed" : "" + if ("$truth_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf" == "${prefix}.vcf.gz") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf_index" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + if ("$query_vcf_index" == "${prefix}.vcf.gz.tbi") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + + // dotnet /opt/Wittyer/Wittyer.dll might need to be replaced with new docker image + """ + mkdir bench + + dotnet /opt/Wittyer/Wittyer.dll \\ + --truthVcf=${truth_vcf} \\ + --inputVcf=${query_vcf} \\ + --outputDirectory=bench \\ + ${regions} \\ + ${args} + + mv bench/Wittyer.Stats.json ${prefix}.json + mv bench/*.vcf.gz ${prefix}.vcf.gz + mv bench/*.vcf.gz.tbi ${prefix}.vcf.gz.tbi + + rm -rf bench + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wittyer: \$(dotnet /opt/Wittyer/Wittyer.dll --version |& sed '1!d ; s/witty.er //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.json + echo "" | gzip > ${prefix}.vcf.gz + touch ${prefix}.vcf.gz.tbi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wittyer: \$(dotnet /opt/Wittyer/Wittyer.dll --version |& sed '1!d ; s/witty.er //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/wittyer/meta.yml b/modules/nf-core/wittyer/meta.yml new file mode 100644 index 00000000000..097f90fbbad --- /dev/null +++ b/modules/nf-core/wittyer/meta.yml @@ -0,0 +1,66 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: wittyer +description: A large variant benchmarking tool analogous to hap.py for small variants. +keywords: + - structural-variants + - benchmarking + - vcf +tools: + - wittyer: + description: "Illumina tool for large variant benchmarking" + homepage: "https://github.com/Illumina/witty.er" + documentation: "https://github.com/Illumina/witty.er" + tool_dev_url: "https://github.com/Illumina/witty.er" + licence: ["BSD-2"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - query_vcf: + type: file + description: A VCF with called variants to benchmark against the standard + pattern: "*.{vcf,vcf.gz}" + - query_vcf_index: + type: file + description: The index of the called VCF (optional) + pattern: "*.tbi" + - truth_vcf: + type: file + description: A standard VCF to compare against + pattern: "*.{vcf,vcf.gz}" + - truth_vcf_index: + type: file + description: The index of the standard VCF (optional) + pattern: "*.tbi" + - bed: + type: file + description: A BED file specifying regions to be included in the analysis (optional) + pattern: "*.bed" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - report: + type: file + description: Detailed per-sample-pair, per-svtype, per-bin stats + pattern: "*.json" + - bench_vcf: + type: file + description: Updated query and truth entries merged into one file + pattern: "*.vcf.gz" + - bench_vcf_tbi: + type: file + description: Index of merged query and truth entries VCF file + pattern: "*.vcf.gz.tbi" +authors: + - "@famosab" +maintainers: + - "@famosab" diff --git a/modules/nf-core/wittyer/tests/main.nf.test b/modules/nf-core/wittyer/tests/main.nf.test new file mode 100644 index 00000000000..3c23ffe66eb --- /dev/null +++ b/modules/nf-core/wittyer/tests/main.nf.test @@ -0,0 +1,100 @@ +nextflow_process { + + name "Test Process WITTYER" + script "../main.nf" + process "WITTYER" + + tag "modules" + tag "modules_nfcore" + tag "wittyer" + + test("human - simulatedSV - vcf_gz") { + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2_tbi'], checkIfExists: true), + [] + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot(process.out.bench_vcf).match("bench_vcf") }, + { assert snapshot(process.out.bench_vcf_tbi).match("bench_vcf_tbi") }, + { assert snapshot(process.out.report).match("report") }, + { assert snapshot(process.out.version).match("version") } + ) + } + + } + + test("human - simulatedSV - vcf_gz - bed") { + + when { + process { + """ + input[0] = [ + [ id:'test_bed', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot(process.out.bench_vcf).match("bed_bench_vcf") }, + { assert snapshot(process.out.bench_vcf_tbi).match("bed_bench_vcf_tbi") }, + { assert snapshot(process.out.report).match("bed_report") }, + { assert snapshot(process.out.version).match("bed_version") } + ) + } + + } + + test("human - simulatedSV - vcf_gz - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test_stub', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['simulated_sv'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv_tbi'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['simulated_sv2_tbi'], checkIfExists: true), + [] + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot(process.out.version).match("stub_version") } + ) + } + + } + +} diff --git a/modules/nf-core/wittyer/tests/main.nf.test.snap b/modules/nf-core/wittyer/tests/main.nf.test.snap new file mode 100644 index 00000000000..a25d824420d --- /dev/null +++ b/modules/nf-core/wittyer/tests/main.nf.test.snap @@ -0,0 +1,347 @@ +{ + "human - simulatedSV - vcf_gz - bed": { + "content": [ + { + "0": [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.json:md5,c6515ada81b5ccf5aa5b4f1268da2800" + ] + ], + "1": [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz:md5,7e5f24415c80ca986e81be90f831e000" + ] + ], + "2": [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz.tbi:md5,e4de1e1d27208b56f5a7bfbe31542240" + ] + ], + "3": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ], + "bench_vcf": [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz:md5,7e5f24415c80ca986e81be90f831e000" + ] + ], + "bench_vcf_tbi": [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz.tbi:md5,e4de1e1d27208b56f5a7bfbe31542240" + ] + ], + "report": [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.json:md5,c6515ada81b5ccf5aa5b4f1268da2800" + ] + ], + "versions": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:41:49.782336703" + }, + "stub_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:57.223103062" + }, + "human - simulatedSV - vcf_gz": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,fb70eac691c1067167091ab2d3b12de3" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,ff56c3084a59507362f6b7b7dc46ffdc" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,b9b448e5f11eebbcfeb9a123e838caa4" + ] + ], + "3": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ], + "bench_vcf": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,ff56c3084a59507362f6b7b7dc46ffdc" + ] + ], + "bench_vcf_tbi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,b9b448e5f11eebbcfeb9a123e838caa4" + ] + ], + "report": [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,fb70eac691c1067167091ab2d3b12de3" + ] + ], + "versions": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:41:44.466462714" + }, + "bench_vcf": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz:md5,ff56c3084a59507362f6b7b7dc46ffdc" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.523573536" + }, + "bench_vcf_tbi": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test.vcf.gz.tbi:md5,b9b448e5f11eebbcfeb9a123e838caa4" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.532504108" + }, + "report": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test.json:md5,fb70eac691c1067167091ab2d3b12de3" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.537692046" + }, + "bed_report": { + "content": [ + [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.json:md5,c6515ada81b5ccf5aa5b4f1268da2800" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.863563564" + }, + "bed_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.868467701" + }, + "version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:47.542666285" + }, + "human - simulatedSV - vcf_gz - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test_stub", + "single_end": false + }, + "test_stub.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test_stub", + "single_end": false + }, + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "2": [ + [ + { + "id": "test_stub", + "single_end": false + }, + "test_stub.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ], + "bench_vcf": [ + [ + { + "id": "test_stub", + "single_end": false + }, + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "bench_vcf_tbi": [ + [ + { + "id": "test_stub", + "single_end": false + }, + "test_stub.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "report": [ + [ + { + "id": "test_stub", + "single_end": false + }, + "test_stub.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,4a5148f206a3b12f0ebe87e81cedc31a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:57.215084162" + }, + "bed_bench_vcf": { + "content": [ + [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz:md5,7e5f24415c80ca986e81be90f831e000" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.852045028" + }, + "bed_bench_vcf_tbi": { + "content": [ + [ + [ + { + "id": "test_bed", + "single_end": false + }, + "test_bed.vcf.gz.tbi:md5,e4de1e1d27208b56f5a7bfbe31542240" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:36:52.857651771" + } +} \ No newline at end of file diff --git a/modules/nf-core/wittyer/tests/tags.yml b/modules/nf-core/wittyer/tests/tags.yml new file mode 100644 index 00000000000..177db940a55 --- /dev/null +++ b/modules/nf-core/wittyer/tests/tags.yml @@ -0,0 +1,2 @@ +wittyer: + - "modules/nf-core/wittyer/**" From dcf17cc0ed8fd5ea57e61a13e0147cddb5c1ee30 Mon Sep 17 00:00:00 2001 From: dimple-aspiring-cat <148437461+dimple-aspiring-cat@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:39:55 +0100 Subject: [PATCH 12/19] Remove unnecessary .view() in subworkflows/nf-core/vcf_phase_shapeit5 & tests/modules/nf-core/shapeit5/ligate/main.nf (#5280) * .view() deleted * Delete unecessary view * Update licence * Bcftools view compressed output * Update licence --------- Co-authored-by: Dimple Adiwal Co-authored-by: Dimple Adiwal Co-authored-by: Louis LE NEZET <58640615+LouisLeNezet@users.noreply.github.com> --- modules/nf-core/shapeit5/ligate/meta.yml | 4 ++-- subworkflows/nf-core/vcf_phase_shapeit5/main.nf | 3 +-- tests/modules/nf-core/shapeit5/ligate/main.nf | 4 +--- tests/modules/nf-core/shapeit5/ligate/nextflow.config | 1 + tests/subworkflows/nf-core/vcf_phase_shapeit5/nextflow.config | 1 + 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/nf-core/shapeit5/ligate/meta.yml b/modules/nf-core/shapeit5/ligate/meta.yml index ab68274eb89..ed1e5e9efed 100644 --- a/modules/nf-core/shapeit5/ligate/meta.yml +++ b/modules/nf-core/shapeit5/ligate/meta.yml @@ -12,8 +12,8 @@ tools: homepage: "https://odelaneau.github.io/shapeit5/" documentation: "https://odelaneau.github.io/shapeit5/docs/documentation" tool_dev_url: "https://github.com/odelaneau/shapeit5" - doi: "10.1101/2022.10.19.512867 " - licence: "['MIT']" + doi: "10.1101/2022.10.19.512867" + licence: ["MIT"] input: - meta: type: map diff --git a/subworkflows/nf-core/vcf_phase_shapeit5/main.nf b/subworkflows/nf-core/vcf_phase_shapeit5/main.nf index 0ddebcb3d41..966f9019055 100644 --- a/subworkflows/nf-core/vcf_phase_shapeit5/main.nf +++ b/subworkflows/nf-core/vcf_phase_shapeit5/main.nf @@ -66,10 +66,9 @@ workflow VCF_PHASE_SHAPEIT5 { ch_ligate_input = SHAPEIT5_PHASECOMMON.out.phased_variant .join(VCF_INDEX1.out.csi, failOnMismatch:true, failOnDuplicate:true) - .view() .map{ meta, vcf, csi -> newmeta = meta + [id: meta.id.split("_")[0..-2].join("_")] - [newmeta, vcf, csi]}.view() + [newmeta, vcf, csi]} .combine(ch_chunks_number, by:0) .map{meta, vcf, csi, chunks_num -> [groupKey(meta, chunks_num), vcf, csi]} diff --git a/tests/modules/nf-core/shapeit5/ligate/main.nf b/tests/modules/nf-core/shapeit5/ligate/main.nf index 3b92a03ed8f..1213f7bd3a8 100644 --- a/tests/modules/nf-core/shapeit5/ligate/main.nf +++ b/tests/modules/nf-core/shapeit5/ligate/main.nf @@ -32,7 +32,7 @@ workflow test_shapeit5_ligate { phase_input = Channel.of([[ id:'NA12878_1X']]) .combine(BCFTOOLS_VIEW.out.vcf.collect().map{it[1]}) .combine(BCFTOOLS_INDEX.out.csi.collect().map{it[1]}) - .combine(sample).view() + .combine(sample) .combine(region) .map{ meta, vcf, csi, sample, region -> [meta + [region: region.replace(":","_")], @@ -43,7 +43,6 @@ workflow test_shapeit5_ligate { phased_variant = SHAPEIT5_PHASECOMMON.output.phased_variant .map{ meta, vcf -> [meta.subMap(["id"]), vcf]} - .view() BCFTOOLS_INDEX2 ( phased_variant ) @@ -55,6 +54,5 @@ workflow test_shapeit5_ligate { a.getName() <=> b.getName()}, csi]} - ligate_input.view() SHAPEIT5_LIGATE ( ligate_input ) } diff --git a/tests/modules/nf-core/shapeit5/ligate/nextflow.config b/tests/modules/nf-core/shapeit5/ligate/nextflow.config index 665b5b02e5f..f96da6737f4 100644 --- a/tests/modules/nf-core/shapeit5/ligate/nextflow.config +++ b/tests/modules/nf-core/shapeit5/ligate/nextflow.config @@ -1,6 +1,7 @@ process { withName: BCFTOOLS_VIEW { ext.args = [ + "-Oz", "-e 'GT=\"./.\"||GT=\".\"'" ].join(' ') ext.prefix = { "${meta.id}" } diff --git a/tests/subworkflows/nf-core/vcf_phase_shapeit5/nextflow.config b/tests/subworkflows/nf-core/vcf_phase_shapeit5/nextflow.config index 2d06b2b2718..9a9f10a640b 100644 --- a/tests/subworkflows/nf-core/vcf_phase_shapeit5/nextflow.config +++ b/tests/subworkflows/nf-core/vcf_phase_shapeit5/nextflow.config @@ -5,6 +5,7 @@ process { } withName: BCFTOOLS_VIEW { ext.args = [ + "-Oz", "-e 'GT=\"./.\"||GT=\".\"'" ].join(' ') } From 0c3919176959e6bf72fec1d8e1539add87426bb2 Mon Sep 17 00:00:00 2001 From: soulj Date: Wed, 20 Mar 2024 10:16:48 +0000 Subject: [PATCH 13/19] Igv reports (#5263) * add inital igvreports module * add full input and tests --------- Co-authored-by: Andrea Telatin <15690844+telatin@users.noreply.github.com> --- modules/nf-core/igvreports/environment.yml | 9 ++ modules/nf-core/igvreports/main.nf | 48 ++++++++ modules/nf-core/igvreports/meta.yml | 47 ++++++++ modules/nf-core/igvreports/tests/main.nf.test | 107 ++++++++++++++++++ .../igvreports/tests/main.nf.test.snap | 38 +++++++ .../igvreports/tests/nextflow_genome.config | 7 ++ modules/nf-core/igvreports/tests/tags.yml | 2 + 7 files changed, 258 insertions(+) create mode 100644 modules/nf-core/igvreports/environment.yml create mode 100644 modules/nf-core/igvreports/main.nf create mode 100644 modules/nf-core/igvreports/meta.yml create mode 100644 modules/nf-core/igvreports/tests/main.nf.test create mode 100644 modules/nf-core/igvreports/tests/main.nf.test.snap create mode 100644 modules/nf-core/igvreports/tests/nextflow_genome.config create mode 100644 modules/nf-core/igvreports/tests/tags.yml diff --git a/modules/nf-core/igvreports/environment.yml b/modules/nf-core/igvreports/environment.yml new file mode 100644 index 00000000000..5ed725f46e9 --- /dev/null +++ b/modules/nf-core/igvreports/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +name: "igvreports" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::igv-reports=1.12.0" diff --git a/modules/nf-core/igvreports/main.nf b/modules/nf-core/igvreports/main.nf new file mode 100644 index 00000000000..3fbc0c05a92 --- /dev/null +++ b/modules/nf-core/igvreports/main.nf @@ -0,0 +1,48 @@ +process IGVREPORTS { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/igv-reports:1.12.0--pyh7cba7a3_0': + 'biocontainers/igv-reports:1.12.0--pyh7cba7a3_0' }" + + input: + tuple val(meta), path(sites) + path genomeFasta //optional genome fasta file + + output: + tuple val(meta), path("*.html") , emit: report + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def fasta = genomeFasta ? "--fasta ${genomeFasta}" : "" + """ + create_report $sites \ + $args \ + $fasta \ + --output ${meta.id}_report.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + igvreports: \$(python -c "import igv_reports; print(igv_reports.__version__)") + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${meta.id}_report.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + igvreports: \$(python -c "import igv_reports; print(igv_reports.__version__)") + END_VERSIONS + """ +} diff --git a/modules/nf-core/igvreports/meta.yml b/modules/nf-core/igvreports/meta.yml new file mode 100644 index 00000000000..6c1ecfdcc29 --- /dev/null +++ b/modules/nf-core/igvreports/meta.yml @@ -0,0 +1,47 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "igvreports" +description: A Python application to generate self-contained HTML reports for variant review and other genomic applications +keywords: + - vcf + - variant + - genomics +tools: + - "igvreports": + description: "Creates self-contained html pages for visual variant review with IGV (igv.js)." + homepage: "https://github.com/igvteam/igv-reports" + documentation: "https://github.com/igvteam/igv-reports" + tool_dev_url: "https://github.com/igvteam/igv-reports" + doi: "10.1093/bioinformatics/btac830" + licence: ["MIT"] + +input: + # Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + + - sites: + type: file + description: VCF, BED, MAF, BEDPE, or generic tab delimited file of genomic variant sites + - genomeFasta: + type: file + description: Reference fasta file + pattern: "*.{fasta,fa}" + +output: + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - report: + type: file + description: html report with a table of genomic sites and an embedded IGV genome browser for viewing data for each site + pattern: "*.{html}" + +authors: + - "@souljamie" +maintainers: + - "@souljamie" diff --git a/modules/nf-core/igvreports/tests/main.nf.test b/modules/nf-core/igvreports/tests/main.nf.test new file mode 100644 index 00000000000..de6a174cba1 --- /dev/null +++ b/modules/nf-core/igvreports/tests/main.nf.test @@ -0,0 +1,107 @@ +nextflow_process { + + name "Test Process IGVREPORTS" + script "../main.nf" + process "IGVREPORTS" + + tag "modules" + tag "modules_nfcore" + tag "igvreports" + + test("testvariantsWithFasta - vcf") { + + when { + process { + """ + input[0] = [ [ id:'test'], // meta map + file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) + ] + input[1] = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match()}, + { + with(process.out.report) { + with(get(0)) { + assert get(1).endsWith("_report.html") + } + } + } + ) + } + + } + + + + test("testvariantsWithGenome - vcf") { + config "./nextflow_genome.config" + + when { + process { + """ + input[0] = [ + [ id:'test'], // meta map + file("https://raw.githubusercontent.com/igvteam/igv-reports/master/test/data/variants/variants.vcf") + ] + // Need to specify the empty optional fasta input + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match()}, + { + with(process.out.report) { + with(get(0)) { + assert get(1).endsWith("_report.html") + } + } + } + ) + } + + } + + test("testvariantWithGenome - vcf - stub") { + config "./nextflow_genome.config" + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test'], // meta map + file("https://raw.githubusercontent.com/igvteam/igv-reports/master/test/data/variants/variants.vcf") + ] + input[1] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match()}, + { + with(process.out.report) { + with(get(0)) { + assert get(1).endsWith("_report.html") + } + } + } + ) + } + + } + +} diff --git a/modules/nf-core/igvreports/tests/main.nf.test.snap b/modules/nf-core/igvreports/tests/main.nf.test.snap new file mode 100644 index 00000000000..d17d7bf3f9a --- /dev/null +++ b/modules/nf-core/igvreports/tests/main.nf.test.snap @@ -0,0 +1,38 @@ +{ + "testvariantWithGenome - vcf - stub": { + "content": [ + [ + "versions.yml:md5,38325133c4cb8cd794588faf67feb345" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T12:09:37.091895286" + }, + "testvariantsWithGenome - vcf": { + "content": [ + [ + "versions.yml:md5,38325133c4cb8cd794588faf67feb345" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T13:55:24.216860945" + }, + "testvariantsWithFasta - vcf": { + "content": [ + [ + "versions.yml:md5,38325133c4cb8cd794588faf67feb345" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T13:41:37.146728439" + } +} \ No newline at end of file diff --git a/modules/nf-core/igvreports/tests/nextflow_genome.config b/modules/nf-core/igvreports/tests/nextflow_genome.config new file mode 100644 index 00000000000..3bb4626ba23 --- /dev/null +++ b/modules/nf-core/igvreports/tests/nextflow_genome.config @@ -0,0 +1,7 @@ +process { + + withName: IGVREPORTS { + ext.args = '--genome hg38' + } + +} diff --git a/modules/nf-core/igvreports/tests/tags.yml b/modules/nf-core/igvreports/tests/tags.yml new file mode 100644 index 00000000000..51c88459074 --- /dev/null +++ b/modules/nf-core/igvreports/tests/tags.yml @@ -0,0 +1,2 @@ +igvreports: + - "modules/nf-core/igvreports/**" From 53c2b466994f07def210b7f4cc866bb5a8a2cb92 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Wed, 20 Mar 2024 11:19:38 +0100 Subject: [PATCH 14/19] Revert "update kallistobustools count output list" (#5307) Revert "update kallistobustools count output list (#5110)" This reverts commit 9d3e489286eead7dfe1010fd324904d8b698eca7. --- .../nf-core/kallistobustools/count/main.nf | 8 ++--- .../nf-core/kallistobustools/count/meta.yml | 8 ----- .../count/tests/main.nf.test.snap | 30 ++----------------- 3 files changed, 5 insertions(+), 41 deletions(-) diff --git a/modules/nf-core/kallistobustools/count/main.nf b/modules/nf-core/kallistobustools/count/main.nf index 1efda00abb8..841ea2fea6c 100644 --- a/modules/nf-core/kallistobustools/count/main.nf +++ b/modules/nf-core/kallistobustools/count/main.nf @@ -17,11 +17,9 @@ process KALLISTOBUSTOOLS_COUNT { val workflow_mode output: - tuple val(meta), path ("*.count") , emit: count - tuple val(meta), path ("*.count/counts_unfiltered"), emit: raw_counts - tuple val(meta), path ("*.count/counts_filtered") , emit: filtered_counts, optional: true - path "versions.yml" , emit: versions - path "*.count/*/*.mtx" , emit: matrix //Ensure that kallisto finished and produced outputs + tuple val(meta), path ("*.count") , emit: count + path "versions.yml" , emit: versions + path "*.count/*/*.mtx" , emit: matrix //Ensure that kallisto finished and produced outputs when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/kallistobustools/count/meta.yml b/modules/nf-core/kallistobustools/count/meta.yml index d491dffaae3..55d5dc6cdc4 100644 --- a/modules/nf-core/kallistobustools/count/meta.yml +++ b/modules/nf-core/kallistobustools/count/meta.yml @@ -58,14 +58,6 @@ output: type: file description: kb count output folder pattern: "*.{count}" - - raw_counts: - type: file - description: kb raw counts output folder - pattern: "*.{count}/counts_unfiltered" - - filtered_counts: - type: file - description: kb filtered counts output folder - pattern: "*.{count}/counts_filtered" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/kallistobustools/count/tests/main.nf.test.snap b/modules/nf-core/kallistobustools/count/tests/main.nf.test.snap index 6f6b3183328..3378c3c16fa 100644 --- a/modules/nf-core/kallistobustools/count/tests/main.nf.test.snap +++ b/modules/nf-core/kallistobustools/count/tests/main.nf.test.snap @@ -15,22 +15,9 @@ ] ], "1": [ - [ - { - "id": "test" - }, - [ - "cells_x_genes.mtx:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ] - ], - "2": [ - - ], - "3": [ "versions.yml:md5,6ec06270afe0a7572c41567160d927d9" ], - "4": [ + "2": [ "cells_x_genes.mtx:md5,d41d8cd98f00b204e9800998ecf8427e" ], "count": [ @@ -44,23 +31,10 @@ ] ] ] - ], - "filtered_counts": [ - ], "matrix": [ "cells_x_genes.mtx:md5,d41d8cd98f00b204e9800998ecf8427e" ], - "raw_counts": [ - [ - { - "id": "test" - }, - [ - "cells_x_genes.mtx:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ] - ], "versions": [ "versions.yml:md5,6ec06270afe0a7572c41567160d927d9" ] @@ -70,7 +44,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-18T11:38:48.980939376" + "timestamp": "2024-03-01T15:48:45.775904562" }, "genome.fasta + genome.gtf + '10X3' + 'standard'": { "content": [ From 1774f7876ee03f65ccf49ca2e6bdef7c2356ebca Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Wed, 20 Mar 2024 11:19:46 +0100 Subject: [PATCH 15/19] Revert "add paths in output directive in cellranger cout module" (#5306) Revert "add paths in output directive in cellranger cout module (#5108)" This reverts commit 92ca535c5a8c0fe89eb71e649ee536bd355ce4fc. --- modules/nf-core/cellranger/count/main.nf | 10 +--- modules/nf-core/cellranger/count/meta.yml | 8 --- .../cellranger/count/tests/main.nf.test.snap | 52 +------------------ 3 files changed, 4 insertions(+), 66 deletions(-) diff --git a/modules/nf-core/cellranger/count/main.nf b/modules/nf-core/cellranger/count/main.nf index 1811d7457ff..42aa09c9f5b 100644 --- a/modules/nf-core/cellranger/count/main.nf +++ b/modules/nf-core/cellranger/count/main.nf @@ -9,10 +9,8 @@ process CELLRANGER_COUNT { path reference output: - tuple val(meta), path("**/outs/**") , emit: outs - tuple val(meta), path("**/outs/filtered_feature_bc_matrix**"), emit: filtered - tuple val(meta), path("**/outs/raw_feature_bc_matrix**") , emit: raw - path "versions.yml" , emit: versions + tuple val(meta), path("**/outs/**"), emit: outs + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -34,11 +32,7 @@ process CELLRANGER_COUNT { def prefix = task.ext.prefix ?: "${meta.id}" """ mkdir -p "${prefix}/outs/" - mkdir -p "${prefix}/outs/filtered_feature_bc_matrix" - mkdir -p "${prefix}/outs/raw_feature_bc_matrix" echo "$prefix" > ${prefix}/outs/fake_file.txt - echo "$prefix" > ${prefix}/outs/filtered_feature_bc_matrix/fake_file.txt - echo "$prefix" > ${prefix}/outs/raw_feature_bc_matrix/fake_file.txt cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/cellranger/count/meta.yml b/modules/nf-core/cellranger/count/meta.yml index 562449769b8..1f1768a857b 100644 --- a/modules/nf-core/cellranger/count/meta.yml +++ b/modules/nf-core/cellranger/count/meta.yml @@ -40,14 +40,6 @@ output: type: file description: Files containing the outputs of Cell Ranger, see official 10X Genomics documentation for a complete list pattern: "${meta.id}/outs/*" - - filtered: - type: file - description: Files containing the filtered outputs of Cell Ranger. - pattern: "**/outs/filtered_feature_bc_matrix**" - - raw: - type: file - description: Files containing the raw outputs of Cell Ranger. - pattern: "**/outs/raw_feature_bc_matrix**" - versions: type: file description: File containing software version diff --git a/modules/nf-core/cellranger/count/tests/main.nf.test.snap b/modules/nf-core/cellranger/count/tests/main.nf.test.snap index edfb304b733..7eafafd02e9 100644 --- a/modules/nf-core/cellranger/count/tests/main.nf.test.snap +++ b/modules/nf-core/cellranger/count/tests/main.nf.test.snap @@ -27,30 +27,6 @@ "content": [ { "0": [ - [ - { - "id": "test_10x", - "single_end": false, - "strandedness": "auto" - }, - [ - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d", - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d", - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d" - ] - ] - ], - "1": [ - [ - { - "id": "test_10x", - "single_end": false, - "strandedness": "auto" - }, - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d" - ] - ], - "2": [ [ { "id": "test_10x", @@ -60,34 +36,10 @@ "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d" ] ], - "3": [ + "1": [ "versions.yml:md5,30cee1a9146b01c48d9b1db6bbe813b6" ], - "filtered": [ - [ - { - "id": "test_10x", - "single_end": false, - "strandedness": "auto" - }, - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d" - ] - ], "outs": [ - [ - { - "id": "test_10x", - "single_end": false, - "strandedness": "auto" - }, - [ - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d", - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d", - "fake_file.txt:md5,0d98223c768861fd6af96f00148dbb8d" - ] - ] - ], - "raw": [ [ { "id": "test_10x", @@ -106,6 +58,6 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-03-18T11:41:17.258523741" + "timestamp": "2024-03-05T17:16:12.322822411" } } \ No newline at end of file From 9d0f89b445e1f5b2fb30476f4be9a8b519c07846 Mon Sep 17 00:00:00 2001 From: Oliver Schwengers Date: Wed, 20 Mar 2024 11:52:05 +0100 Subject: [PATCH 16/19] Remove AMRFinderPlus DB update on each invocation (#5232) Remove the AMRFinderPlus DB update which is invoked on each Bakta execution, nevertheless, it should be downloaded and installed during the DB download via bakta_db download. Co-authored-by: Jasmin Frangenberg <73216762+jasmezz@users.noreply.github.com> --- modules/nf-core/bakta/bakta/main.nf | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/nf-core/bakta/bakta/main.nf b/modules/nf-core/bakta/bakta/main.nf index 7c83d94bf19..9a32c3dac93 100644 --- a/modules/nf-core/bakta/bakta/main.nf +++ b/modules/nf-core/bakta/bakta/main.nf @@ -35,8 +35,6 @@ process BAKTA_BAKTA { def proteins_opt = proteins ? "--proteins ${proteins[0]}" : "" def prodigal_tf = prodigal_tf ? "--prodigal-tf ${prodigal_tf[0]}" : "" """ - amrfinder_update --force_update --database $db/amrfinderplus-db - bakta \\ $fasta \\ $args \\ From 9f892b5da34b25da790af7baa48caeb869481bb9 Mon Sep 17 00:00:00 2001 From: Christian Heide <11767963+jch-13@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:02:00 +0100 Subject: [PATCH 17/19] Add subworkflow mapAD (#5239) Add mapAD subworkflow --- .../nf-core/fastq_align_mapad/main.nf | 76 +++++++++++++ .../nf-core/fastq_align_mapad/meta.yml | 100 ++++++++++++++++++ .../fastq_align_mapad/tests/main.nf.test | 67 ++++++++++++ .../fastq_align_mapad/tests/main.nf.test.snap | 66 ++++++++++++ .../fastq_align_mapad/tests/nextflow.config | 12 +++ .../nf-core/fastq_align_mapad/tests/tags.yml | 2 + 6 files changed, 323 insertions(+) create mode 100644 subworkflows/nf-core/fastq_align_mapad/main.nf create mode 100644 subworkflows/nf-core/fastq_align_mapad/meta.yml create mode 100644 subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test create mode 100644 subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test.snap create mode 100644 subworkflows/nf-core/fastq_align_mapad/tests/nextflow.config create mode 100644 subworkflows/nf-core/fastq_align_mapad/tests/tags.yml diff --git a/subworkflows/nf-core/fastq_align_mapad/main.nf b/subworkflows/nf-core/fastq_align_mapad/main.nf new file mode 100644 index 00000000000..6c114892f3e --- /dev/null +++ b/subworkflows/nf-core/fastq_align_mapad/main.nf @@ -0,0 +1,76 @@ +// +// Alignment with mapAD and sort +// + +include { MAPAD_MAP } from '../../../modules/nf-core/mapad/map/main' +include { BAM_SORT_STATS_SAMTOOLS } from '../bam_sort_stats_samtools/main' + +workflow FASTQ_ALIGN_MAPAD { + + take: + ch_reads // channel (mandatory): [ val(meta), path(reads) ]. Important: meta REQUIRES single_end` entry! + ch_index // channel (mandatory): [ val(meta), path(index) ] + ch_fasta // channel (optional) : [ val(meta3), path(fasta) ] + val_mismatch_parameter + val_double_stranded_library + val_five_prime_overhang + val_three_prime_overhang + val_deam_rate_double_stranded + val_deam_rate_single_stranded + val_indel_rate + + + main: + + ch_versions = Channel.empty() + + // WARNING: You must specify in your prefix `meta.id_index` in your `modules.conf` + // to ensure that you do not overwrite multiple BAM files from one sample mapped + // against multiple references. This meta map is added by the subworkflow but can be removed + // after if necessary. + + // Ensure when multiple references are provided, that reference/read combinations + // are correctly associated throughout the subworkflow by copying the sample + // specific metadata to the index on each combination + + ch_prepped_input = ch_reads + .combine(ch_index) + .map{ + meta, reads, meta_index, index -> + + // Create a combined id that includes the ids of the reads and the index used. + // Also keep the id of the index with a new name to avoid name collisions. + def key_read_ref = meta.id + "_" + meta_index.id + def id_index = meta_index.id + + [ meta + [key_read_ref: key_read_ref] + [id_index: id_index], reads, meta_index + [key_read_ref: key_read_ref] + [id_index: id_index], index ] + } + + // Drop the index_meta, as the id of the index is now kept within the read meta. + ch_preppedinput_for_mapad = ch_prepped_input + .multiMap { + meta, reads, meta_index, index -> + reads: [ meta, reads ] + index: [ meta, index ] + } + + // Alignment + MAPAD_MAP ( ch_preppedinput_for_mapad.reads, ch_preppedinput_for_mapad.index, val_mismatch_parameter, val_double_stranded_library, val_five_prime_overhang, val_three_prime_overhang, val_deam_rate_double_stranded, val_deam_rate_single_stranded, val_indel_rate ) + ch_versions = ch_versions.mix( MAPAD_MAP.out.versions.first() ) + + // Sort, index BAM file and run samtools stats, flagstat and idxstats + BAM_SORT_STATS_SAMTOOLS ( MAPAD_MAP.out.bam, ch_fasta ) + ch_versions = ch_versions.mix(BAM_SORT_STATS_SAMTOOLS.out.versions) + + emit: + bam_unsorted = MAPAD_MAP.out.bam // channel: [ val(meta), path(bam) ] + + bam = BAM_SORT_STATS_SAMTOOLS.out.bam // channel: [ val(meta), path(bam) ] + bai = BAM_SORT_STATS_SAMTOOLS.out.bai // channel: [ val(meta), path(bai) ] + csi = BAM_SORT_STATS_SAMTOOLS.out.csi // channel: [ val(meta), path(csi) ] + stats = BAM_SORT_STATS_SAMTOOLS.out.stats // channel: [ val(meta), path(stats) ] + flagstat = BAM_SORT_STATS_SAMTOOLS.out.flagstat // channel: [ val(meta), path(flagstat) ] + idxstats = BAM_SORT_STATS_SAMTOOLS.out.idxstats // channel: [ val(meta), path(idxstats) ] + + versions = ch_versions // channel: [ path(versions.yml) ] +} diff --git a/subworkflows/nf-core/fastq_align_mapad/meta.yml b/subworkflows/nf-core/fastq_align_mapad/meta.yml new file mode 100644 index 00000000000..c459fd263e9 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_mapad/meta.yml @@ -0,0 +1,100 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "fastq_align_mapad" +description: Align FASTQ files against reference genome with the mapAD aDNA short-read aligner producing a sorted and indexed BAM files +keywords: + - sort + - fastq + - bam + - mapad + - align + - map +components: + - mapad/map + - samtools/sort + - samtools/index + - samtools/stats + - samtools/idxstats + - samtools/flagstat + - bam_sort_stats_samtools +input: + - ch_reads: + description: | + List of input FASTQ file + Structure: [ val(meta), path(reads) ] + - ch_index: + description: | + mapAD genome index files + Structure: [ val(meta), path(index) ] + - ch_fasta: + type: file + description: | + Reference fasta file + Structure: [ val(meta), path(fasta) ] + - val_mismatch_parameter: + type: float + description: | + `bwa aln` compatible allowed-mismatches parameter + - val_double_stranded_library: + type: boolean + description: | + If true, `--library` is set to `double_stranded` + pattern: "true|false" + - val_five_prime_overhang: + type: float + description: | + 5' overhang parameter (global overhang parameter + if `val_double_stranded_library` is set to `true`) + - val_three_prime_overhang: + type: float + description: | + 3' overhang parameter (ignored if + `val_double_stranded_library` is set to `true`) + - val_deam_rate_double_stranded: + type: float + description: | + `-d` parameter. Specifies the expected deamination + rate in double-stranded stems of the reads. + - val_deam_rate_single_stranded: + type: float + description: | + `-s` parameter. Specifies the expected deamination + rate in single-stranded overhangs of the reads. + - val_indel_rate: + type: float + description: | + `-i` parameter. Specifies the expected rate of InDels. +output: + - bam_unsorted: + description: | + BAM file produced by mapAD + Structure: [ val(meta), path(bam) ] + - bam: + description: | + BAM file sorted by samtools + Structure: [ val(meta), path(bam) ] + - bai: + description: | + BAI index of the sorted BAM file + Structure: [ val(meta), path(bai) ] + - csi: + description: | + CSI index of the sorted BAM file + Structure: [ val(meta), path(csi) ] + - stats: + description: | + File containing samtools stats output + Structure: [ val(meta), path(stats) ] + - flagstat: + description: | + File containing samtools flagstat output + Structure: [ val(meta), path(flagstat) ] + - idxstats: + description: | + File containing samtools idxstats output + Structure: [ val(meta), path(idxstats) ] + - versions: + description: | + Files containing software versions + Structure: [ path(versions.yml) ] +authors: + - "@jch-13" diff --git a/subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test b/subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test new file mode 100644 index 00000000000..841497c0f93 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test @@ -0,0 +1,67 @@ +nextflow_workflow { + + name "Test Subworkflow FASTQ_ALIGN_MAPAD" + script "../main.nf" + config "./nextflow.config" + workflow "FASTQ_ALIGN_MAPAD" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fastq_align_mapad" + tag "subworkflows/bam_sort_stats_samtools" + tag "mapad" + tag "mapad/map" + tag "mapad/index" + + + test("sarscov2 - bam - single_end") { + setup { + run("MAPAD_INDEX") { + script "../../../../modules/nf-core/mapad/index/main.nf" + process { + """ + input[0] = [ [ id:'genome' ],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + """ + } + } + } + + when { + workflow { + """ + input[0] = Channel.of([ [ id:'test', single_end:true ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + ]) + input[1] = MAPAD_INDEX.out.index + input[2] = [ [ id:'genome' ], + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + input[3] = 0.03 + input[4] = false + input[5] = 0.5 + input[6] = 0.5 + input[7] = 0.02 + input[8] = 1.0 + input[9] = 0.001 + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.bai, + workflow.out.csi, + workflow.out.stats, + workflow.out.flagstat, + workflow.out.idxstats, + workflow.out.versions, + ).match() + }, + { assert file(workflow.out.bam_unsorted.get(0).get(1)).exists() }, + { assert file(workflow.out.bam.get(0).get(1)).exists() }, + ) + } + } +} diff --git a/subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test.snap new file mode 100644 index 00000000000..b069d429bd1 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_mapad/tests/main.nf.test.snap @@ -0,0 +1,66 @@ +{ + "sarscov2 - bam - single_end": { + "content": [ + [ + [ + { + "id": "test", + "single_end": true, + "key_read_ref": "test_genome", + "id_index": "genome" + }, + "test.sorted.bam.bai:md5,4ddb73895b1424a09cab389d536f3611" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true, + "key_read_ref": "test_genome", + "id_index": "genome" + }, + "test.sorted.bam.stats:md5,aa0845cab33b7fed9ce65143fafc4ac6" + ] + ], + [ + [ + { + "id": "test", + "single_end": true, + "key_read_ref": "test_genome", + "id_index": "genome" + }, + "test.sorted.bam.flagstat:md5,08dd9ecd07c50d038756da90d0813961" + ] + ], + [ + [ + { + "id": "test", + "single_end": true, + "key_read_ref": "test_genome", + "id_index": "genome" + }, + "test.sorted.bam.idxstats:md5,4cf38c87984783a24c2ebeb0598292f8" + ] + ], + [ + "versions.yml:md5,3b366ad42150e9e0121caeb0291c1b3c", + "versions.yml:md5,486dcd9469c7e98c54cbcb25651024ea", + "versions.yml:md5,75ff2e03627b2c64930703e205d338ab", + "versions.yml:md5,886ec6d544b4770bc49470fe38689453", + "versions.yml:md5,e96fa5cf98e9910e2680fda162305a1a", + "versions.yml:md5,f1d10c97aa161f4fe262469b7aeb349d" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:55:17.646783154" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_mapad/tests/nextflow.config b/subworkflows/nf-core/fastq_align_mapad/tests/nextflow.config new file mode 100644 index 00000000000..7927c34294a --- /dev/null +++ b/subworkflows/nf-core/fastq_align_mapad/tests/nextflow.config @@ -0,0 +1,12 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + + withName: '.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*' { + ext.prefix = { "${meta.id}.sorted" } + } + + withName: '.*:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' { + ext.prefix = { "${meta.id}.sorted.bam" } + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_mapad/tests/tags.yml b/subworkflows/nf-core/fastq_align_mapad/tests/tags.yml new file mode 100644 index 00000000000..86acf871fa6 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_mapad/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/fastq_align_mapad: + - subworkflows/nf-core/fastq_align_mapad/** From 6920a611782d056e3c11f0d32bb93cfae064524d Mon Sep 17 00:00:00 2001 From: lgrochowalski Date: Wed, 20 Mar 2024 13:03:53 +0100 Subject: [PATCH 18/19] Leviosam2 index (#5316) * first version * improvments * whitespaces * whitespaces * whitespaces * prettier --------- Co-authored-by: zxBIB Grochowalski,Lukasz (GCBDS) EXTERNAL --- .../nf-core/leviosam2/index/environment.yml | 8 ++ modules/nf-core/leviosam2/index/main.nf | 49 ++++++++++++ modules/nf-core/leviosam2/index/meta.yml | 45 +++++++++++ .../leviosam2/index/tests/main.nf.test | 75 +++++++++++++++++++ .../leviosam2/index/tests/main.nf.test.snap | 68 +++++++++++++++++ .../nf-core/leviosam2/index/tests/tags.yml | 3 + 6 files changed, 248 insertions(+) create mode 100644 modules/nf-core/leviosam2/index/environment.yml create mode 100644 modules/nf-core/leviosam2/index/main.nf create mode 100644 modules/nf-core/leviosam2/index/meta.yml create mode 100644 modules/nf-core/leviosam2/index/tests/main.nf.test create mode 100644 modules/nf-core/leviosam2/index/tests/main.nf.test.snap create mode 100644 modules/nf-core/leviosam2/index/tests/tags.yml diff --git a/modules/nf-core/leviosam2/index/environment.yml b/modules/nf-core/leviosam2/index/environment.yml new file mode 100644 index 00000000000..f59fd991b8f --- /dev/null +++ b/modules/nf-core/leviosam2/index/environment.yml @@ -0,0 +1,8 @@ +--- +name: "leviosam2_index" +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "bioconda::leviosam2=0.4.2" diff --git a/modules/nf-core/leviosam2/index/main.nf b/modules/nf-core/leviosam2/index/main.nf new file mode 100644 index 00000000000..f8ef233bcd1 --- /dev/null +++ b/modules/nf-core/leviosam2/index/main.nf @@ -0,0 +1,49 @@ +process LEVIOSAM2_INDEX { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/leviosam2:0.4.2--h4ac6f70_0': + 'biocontainers/leviosam2:0.4.2--h4ac6f70_0' }" + + input: + tuple val(meta), path(fai) + path(chain) + + output: + tuple val(meta), path("*.clft"), emit: clft + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + leviosam2 \\ + index \\ + -c ${chain} \\ + -p ${prefix} \\ + -F ${fai} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + leviosam2: \$(leviosam2 --version) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.clft + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + leviosam2: \$(leviosam2 --version) + END_VERSIONS + """ +} diff --git a/modules/nf-core/leviosam2/index/meta.yml b/modules/nf-core/leviosam2/index/meta.yml new file mode 100644 index 00000000000..ab5a3a279b2 --- /dev/null +++ b/modules/nf-core/leviosam2/index/meta.yml @@ -0,0 +1,45 @@ +--- +name: "leviosam2_index" +description: Index chain files for lift over +keywords: + - leviosam2 + - index + - lift +tools: + - "leviosam2": + description: "Fast and accurate coordinate conversion between assemblies" + homepage: "https://github.com/milkschen/leviosam2/blob/main/workflow/README.md" + documentation: "https://github.com/milkschen/leviosam2/blob/main/workflow/README.md" + licence: ["MIT"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'reference' ]` + - fai: + type: file + description: FAI (FASTA index) file of the target reference + pattern: "*.{fai}" + - chain: + type: file + description: Chain file to index. + pattern: "*.{chain}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'reference' ]` + - clft: + type: file + description: Clft file of indexed chain + pattern: "*.{clft}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@lgrochowalski" +maintainers: + - "@lgrochowalski" diff --git a/modules/nf-core/leviosam2/index/tests/main.nf.test b/modules/nf-core/leviosam2/index/tests/main.nf.test new file mode 100644 index 00000000000..f1ad0a3b18e --- /dev/null +++ b/modules/nf-core/leviosam2/index/tests/main.nf.test @@ -0,0 +1,75 @@ +nextflow_process { + + name "Test Process LEVIOSAM2_INDEX" + script "../main.nf" + process "LEVIOSAM2_INDEX" + + tag "modules" + tag "modules_nfcore" + tag "leviosam2" + tag "leviosam2/index" + tag "gunzip" + + test("index") { + setup { + run("GUNZIP") { + script "../../../gunzip/main.nf" + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['homo_sapiens']['genome']['genome_chain_gz'], checkIfExists: true) + ] + """ + } + + } + } + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ] + input[1] = GUNZIP.out.gunzip.map{ meta, gunzip -> gunzip } + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("index - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ] + input[1] = file(params.test_data['homo_sapiens']['genome']['genome_chain_gz'], checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/leviosam2/index/tests/main.nf.test.snap b/modules/nf-core/leviosam2/index/tests/main.nf.test.snap new file mode 100644 index 00000000000..d0e9f33ccc8 --- /dev/null +++ b/modules/nf-core/leviosam2/index/tests/main.nf.test.snap @@ -0,0 +1,68 @@ +{ + "index - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.clft:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,36cf8d4dceb8156f779cd6352eba61dc" + ], + "clft": [ + [ + { + "id": "test" + }, + "test.clft:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,36cf8d4dceb8156f779cd6352eba61dc" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T11:21:32.906054" + }, + "index": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.clft:md5,f6ce975bfe33512114ac2cb8aefbffe2" + ] + ], + "1": [ + "versions.yml:md5,36cf8d4dceb8156f779cd6352eba61dc" + ], + "clft": [ + [ + { + "id": "test" + }, + "test.clft:md5,f6ce975bfe33512114ac2cb8aefbffe2" + ] + ], + "versions": [ + "versions.yml:md5,36cf8d4dceb8156f779cd6352eba61dc" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T11:20:40.461365" + } +} \ No newline at end of file diff --git a/modules/nf-core/leviosam2/index/tests/tags.yml b/modules/nf-core/leviosam2/index/tests/tags.yml new file mode 100644 index 00000000000..a1547a1d9dd --- /dev/null +++ b/modules/nf-core/leviosam2/index/tests/tags.yml @@ -0,0 +1,3 @@ +leviosam2/index: + - "modules/nf-core/leviosam2/index/**" + - "modules/nf-core/gunzip/**" From 7b29d1b22c0d7d4d5f03f3295d310e785fd3cb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= <86408271+kbestak@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:13:32 +0100 Subject: [PATCH 19/19] Added contrast limited adaptive histogram equalization module (#5268) * clahe_module * removed todos * Fixed linting, excluded conda tests. * Added conda test skip to workflows/test.yml. --------- Co-authored-by: Florian Wuennemann --- .github/workflows/test.yml | 2 + modules/nf-core/molkartgarage/clahe/main.nf | 49 +++++++++++++++ modules/nf-core/molkartgarage/clahe/meta.yml | 52 ++++++++++++++++ .../molkartgarage/clahe/tests/main.nf.test | 60 +++++++++++++++++++ .../clahe/tests/main.nf.test.snap | 14 +++++ .../molkartgarage/clahe/tests/nextflow.config | 7 +++ .../molkartgarage/clahe/tests/tags.yml | 2 + 7 files changed, 186 insertions(+) create mode 100644 modules/nf-core/molkartgarage/clahe/main.nf create mode 100644 modules/nf-core/molkartgarage/clahe/meta.yml create mode 100644 modules/nf-core/molkartgarage/clahe/tests/main.nf.test create mode 100644 modules/nf-core/molkartgarage/clahe/tests/main.nf.test.snap create mode 100644 modules/nf-core/molkartgarage/clahe/tests/nextflow.config create mode 100644 modules/nf-core/molkartgarage/clahe/tests/tags.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 544b2cf26b1..6bf82516a05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -561,6 +561,8 @@ jobs: tags: merquryfk/merquryfk - profile: conda tags: merquryfk/ploidyplot + - profile: conda + tags: molkartgarage/clahe - profile: conda tags: quartonotebook - profile: conda diff --git a/modules/nf-core/molkartgarage/clahe/main.nf b/modules/nf-core/molkartgarage/clahe/main.nf new file mode 100644 index 00000000000..1d070aeebcf --- /dev/null +++ b/modules/nf-core/molkartgarage/clahe/main.nf @@ -0,0 +1,49 @@ +process MOLKARTGARAGE_CLAHE { + tag "$meta.id" + label 'process_medium' + + container "ghcr.io/schapirolabor/molkart-local:v0.1.1" + + input: + tuple val(meta), path(image) + + output: + tuple val(meta), path("*.tiff") , emit: img_clahe + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "Molkartgarage/clahe module does not support Conda. Please use Docker / Singularity / Podman instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + python /local/scripts/molkart_clahe.py \ + --input ${image} \ + --output ${prefix}.tiff \ + $args + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + molkart_clahe: \$(python /local/scripts/molkart_clahe.py --version) + scikit-image: 0.19.2 + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.tiff + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + molkart_clahe: \$(python3 /local/scripts/molkart_clahe.py --version) + scikit-image: 0.19.2 + END_VERSIONS + """ +} diff --git a/modules/nf-core/molkartgarage/clahe/meta.yml b/modules/nf-core/molkartgarage/clahe/meta.yml new file mode 100644 index 00000000000..9ca71304aa3 --- /dev/null +++ b/modules/nf-core/molkartgarage/clahe/meta.yml @@ -0,0 +1,52 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "molkartgarage_clahe" +description: Contrast-limited adjusted histogram equalization (CLAHE) on single-channel tif images. +keywords: + - clahe + - image_processing + - imaging + - correction +tools: + - "molkartgarage": + description: "One-stop-shop for scripts and tools for processing data for molkart and spatial omics pipelines." + homepage: https://github.com/SchapiroLabor/molkart-local/tree/main + documentation: https://github.com/SchapiroLabor/molkart-local/tree/main + tool_dev_url: https://github.com/SchapiroLabor/molkart-local/blob/main/scripts/molkart_clahe.py + licence: ["MIT"] + +input: + # Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1']` + + - image: + type: file + description: Single-channel tiff file to be corrected. + pattern: "*.{tif,tiff}" + +output: + #Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1']` + + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + + - img_clahe: + type: file + description: CLAHE corrected tiff file. + pattern: "*.{tiff}" + +authors: + - "@kbestak" +maintainers: + - "@kbestak" diff --git a/modules/nf-core/molkartgarage/clahe/tests/main.nf.test b/modules/nf-core/molkartgarage/clahe/tests/main.nf.test new file mode 100644 index 00000000000..15c67b7d7fa --- /dev/null +++ b/modules/nf-core/molkartgarage/clahe/tests/main.nf.test @@ -0,0 +1,60 @@ +// test with: nf-core modules test molkartgarage/clahe +nextflow_process { + + name "Test Process MOLKARTGARAGE_CLAHE" + script "../main.nf" + config "./nextflow.config" + process "MOLKARTGARAGE_CLAHE" + + tag "modules" + tag "modules_nfcore" + tag "molkartgarage" + tag "molkartgarage/clahe" + + test("clahe - tiff") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['imaging']['tiff']['mouse_heart_wga'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.img_clahe }, // uuid in metadata changes so md5sums are not the same + { assert snapshot(process.out.versions).match("versions") } + ) + } + + } + + test("clahe - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['imaging']['tiff']['mouse_heart_wga'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success } + ) + } + + } + +} diff --git a/modules/nf-core/molkartgarage/clahe/tests/main.nf.test.snap b/modules/nf-core/molkartgarage/clahe/tests/main.nf.test.snap new file mode 100644 index 00000000000..349a0e9bfa6 --- /dev/null +++ b/modules/nf-core/molkartgarage/clahe/tests/main.nf.test.snap @@ -0,0 +1,14 @@ +{ + "versions": { + "content": [ + [ + "versions.yml:md5,ddf084b777e987eb83271f02e4e4cf2b" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T13:20:00.607838647" + } +} \ No newline at end of file diff --git a/modules/nf-core/molkartgarage/clahe/tests/nextflow.config b/modules/nf-core/molkartgarage/clahe/tests/nextflow.config new file mode 100644 index 00000000000..72dd3fcceec --- /dev/null +++ b/modules/nf-core/molkartgarage/clahe/tests/nextflow.config @@ -0,0 +1,7 @@ +process { + + withName: "MOLKARTGARAGE_CLAHE" { + ext.args = '--tile-size 368 --tile-size 48 --pixel-size 0.138 --cliplimit 0.01' + } + +} diff --git a/modules/nf-core/molkartgarage/clahe/tests/tags.yml b/modules/nf-core/molkartgarage/clahe/tests/tags.yml new file mode 100644 index 00000000000..b7bba012974 --- /dev/null +++ b/modules/nf-core/molkartgarage/clahe/tests/tags.yml @@ -0,0 +1,2 @@ +molkartgarage/clahe: + - "modules/nf-core/molkartgarage/clahe/**"