Merge pull request #138 from CenterForMedicalGeneticsGhent/dev

Release v1.3.0
nf-cmgg · Jul 10, 2023 · 1e8a16e · 1e8a16e
2 parents 22fd02a + 588c6a9
commit 1e8a16e
Show file tree

Hide file tree

Showing 63 changed files with 1,662 additions and 1,447 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -8,7 +8,7 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = space
 
-[*.{md,yml,yaml,html,css,scss,js,cff}]
+[*.{md,yml,yaml,html,css,scss,js}]
 indent_size = 2
 
 # These files are edited and tested upstream in nf-core/modules

diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -110,4 +110,3 @@ To get started:
 Devcontainer specs:
 
 - [DevContainer config](.devcontainer/devcontainer.json)
-- [Dockerfile](.devcontainer/Dockerfile)
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -38,13 +38,14 @@ body:
     id: system
     attributes:
       label: System information
-      description: "* Nextflow version _(eg. 22.10.1)_
+      description: "* Nextflow version _(eg. 23.04.0)_
 
         * Hardware _(eg. HPC, Desktop, Cloud)_
 
         * Executor _(eg. slurm, local, awsbatch)_
 
-        * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_
+        * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud,
+        or Apptainer)_
 
         * OS _(eg. CentOS Linux, macOS, Linux Mint)_
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "22.10.5"
+          - "23.04.0"
           - "latest-everything"
         test:
           - "default"
@@ -34,9 +34,12 @@ jobs:
           - "gemini_test"
           - "full_wes"
           - "full_wgs"
+          - "only_call"
     steps:
       - name: Free some space
         run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 

diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml
@@ -0,0 +1,24 @@
+name: "Close user-tagged issues and PRs"
+on:
+  schedule:
+    - cron: "0 0 * * 0" # Once a week
+
+jobs:
+  clean-up:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v7
+        with:
+          stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days."
+          stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful."
+          close-issue-message: "This issue was closed because it has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor and then staled for 20 days with no activity."
+          days-before-stale: 30
+          days-before-close: 20
+          days-before-pr-close: -1
+          any-of-labels: "awaiting-changes,awaiting-feedback"
+          exempt-issue-labels: "WIP"
+          exempt-pr-labels: "WIP"
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,4 +1,9 @@
 image: nfcore/gitpod:latest
+tasks:
+  - name: Update Nextflow and setup pre-commit
+    command: |
+      pre-commit install --install-hooks
+      nextflow self-update
 
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack

diff --git a/.nf-core.yml b/.nf-core.yml
@@ -17,6 +17,11 @@ lint:
   readme:
     - nextflow_badge
   files_unchanged:
+    - CODE_OF_CONDUCT.md
+    - assets/nf-core-nf-cmgg-germline_logo_light.png
+    - docs/images/nf-core-nf-cmgg-germline_logo_light.png
+    - docs/images/nf-core-nf-cmgg-germline_logo_dark.png
+    - .github/ISSUE_TEMPLATE/bug_report.yml
     - .github/CONTRIBUTING.md
     - .github/ISSUE_TEMPLATE/bug_report.yml
     - .github/ISSUE_TEMPLATE/feature_request.yml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v2.7.1"
+    hooks:
+      - id: prettier
diff --git a/.prettierignore b/.prettierignore
@@ -14,3 +14,4 @@ samplesheet*
 *.ped
 null
 bin/
+CITATION.cff
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,25 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v1.3.0 - Happy Hasselt - [July 10 2023]
+
+### New Features
+
+1. Added the `--only_call` parameter. Specifying this parameter tells the pipeline to only do variant calling and skip all post-processing. This will only output the GVCFs and files created to help variant calling.
+2. The samplesheet is now also in the output folder.
+3. Added an option `--only_merge` to tell the pipeline to create genomicsdbs and stop running there
+4. Get regions from the GVCF instead of CRAM for joint genotyping. This removes the need to supply a CRAM file when a GVCF file has been used as input.
+
+### Improvements
+
+1. Updated `nf-validation` to v0.2.1.
+2. Updated the samtools/merge tool to the nf-core version. This increases the efficiency and disk space usage of the tool.
+
+### Fixes
+
+1. Fixed an error where the truth VCFs caused a join error when the same sample was given multiple times
+2. Updated some outdated error messages
+
 ## v1.2.2 - Benign Brussels - [June 12 2023]
 
 ### Improvements

diff --git a/CITATIONS.md b/CITATIONS.md
@@ -24,7 +24,10 @@
 
 - [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241)
 
+  > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241.
+
 - [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/)
+
   > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675.
 
 ## Pipeline tools (in alphabetical order)

diff --git a/README.md b/README.md
@@ -20,30 +20,29 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool
 1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.5`)
 2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_.
 
-3. Download the pipeline and test it on a minimal dataset with a single command:
+`samplesheet.csv`:
 
-   ```bash
-   nextflow run CenterForMedicalGeneticsGhent/nf-cmgg-germline -profile test,YOURPROFILE --outdir <OUTDIR>
-   ```
-
-   Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string.
+```csv
+sample,fastq_1,fastq_2
+CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz
+```
 
-   > - The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`.
-   > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile <institute>` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment.
-   > - If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs.
-   > - If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs.
+Each row represents a fastq file (single-end) or a pair of fastq files (paired end).
 
-4. Start running your own analysis!
+-->
 
-   ```bash
-   nextflow run CenterForMedicalGeneticsGhent/nf-cmgg-germline --input <INPUT_CSV/TSV/YAML> --outdir <OUTDIR> -profile <docker/singularity/podman/shifter/charliecloud/conda/institute> --fasta <PATH/TO/FASTA>
-   ```
+Now, you can run the pipeline using:
 
-An overview of the parameters for this pipeline can be viewed at the [parameter docs](docs/parameters.md) or using:
+   <!-- TODO nf-core: Update the example "typical command" below used to run the pipeline -->
 
+```bash
+nextflow run CenterForMedicalGeneticsGhent/nf-cmgg-germline --input samplesheet.csv --outdir <OUTDIR> --genome GRCh37 -profile <docker/singularity/podman/shifter/charliecloud/conda/institute>
 ```
-nextflow run CenterForMedicalGeneticsGhent/nf-cmgg-germline --help
-```
+
+> **Warning:**
+> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those
+> provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
+> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
 
 ## Credits
 

diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml
@@ -5,13 +5,17 @@ section_href: "https://github.com/CenterForMedicalGeneticsGhent/nf-cmgg-germline
 plot_type: "html"
 data: |
   <h4>Methods</h4>
-  <p>Data was processed using CenterForMedicalGeneticsGhent/nf-cmgg-germline v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (<a href="https://doi.org/10.1038/s41587-020-0439-x">Ewels <em>et al.</em>, 2020</a>).</p>
+  <p>Data was processed using CenterForMedicalGeneticsGhent/nf-cmgg-germline v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (<a href="https://doi.org/10.1038/s41587-020-0439-x">Ewels <em>et al.</em>, 2020</a>), utilising reproducible software environments from the Bioconda (<a href="https://doi.org/10.1038/s41592-018-0046-7">Grüning <em>et al.</em>, 2018</a>) and Biocontainers (<a href="https://doi.org/10.1093/bioinformatics/btx192">da Veiga Leprevost <em>et al.</em>, 2017</a>) projects.</p>
   <p>The pipeline was executed with Nextflow v${workflow.nextflow.version} (<a href="https://doi.org/10.1038/nbt.3820">Di Tommaso <em>et al.</em>, 2017</a>) with the following command:</p>
   <pre><code>${workflow.commandLine}</code></pre>
+  <p>${tool_citations}</p>
   <h4>References</h4>
   <ul>
-    <li>Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. <a href="https://doi.org/10.1038/nbt.3820">https://doi.org/10.1038/nbt.3820</a></li>
-    <li>Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. <a href="https://doi.org/10.1038/s41587-020-0439-x">https://doi.org/10.1038/s41587-020-0439-x</a></li>
+    <li>Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. doi: <a href="https://doi.org/10.1038/nbt.3820">10.1038/nbt.3820</a></li>
+    <li>Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. doi: <a href="https://doi.org/10.1038/s41587-020-0439-x">10.1038/s41587-020-0439-x</a></li>
+    <li>Grüning, B., Dale, R., Sjödin, A., Chapman, B. A., Rowe, J., Tomkins-Tinch, C. H., Valieris, R., Köster, J., & Bioconda Team. (2018). Bioconda: sustainable and comprehensive software distribution for the life sciences. Nature Methods, 15(7), 475–476. doi: <a href="https://doi.org/10.1038/s41592-018-0046-7">10.1038/s41592-018-0046-7</a></li>
+    <li>da Veiga Leprevost, F., Grüning, B. A., Alves Aflitos, S., Röst, H. L., Uszkoreit, J., Barsnes, H., Vaudel, M., Moreno, P., Gatto, L., Weber, J., Bai, M., Jimenez, R. C., Sachsenberg, T., Pfeuffer, J., Vera Alvarez, R., Griss, J., Nesvizhskii, A. I., & Perez-Riverol, Y. (2017). BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics (Oxford, England), 33(16), 2580–2582. doi: <a href="https://doi.org/10.1093/bioinformatics/btx192">10.1093/bioinformatics/btx192</a></li>
+    ${tool_bibliography}
   </ul>
   <div class="alert alert-info">
     <h5>Notes:</h5>

diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml
@@ -1,5 +1,5 @@
 report_comment: >
-  This report has been generated by the <a href="https://github.com/CenterForMedicalGeneticsGhent/nf-cmgg-germline" target="_blank">CenterForMedicalGeneticsGhent/nf-cmgg-germline</a>
+  This report has been generated by the <a href="https://github.com/CenterForMedicalGeneticsGhent/nf-cmgg-germline/1.3.0" target="_blank">CenterForMedicalGeneticsGhent/nf-cmgg-germline</a>
   analysis pipeline.
 report_section_order:
   "CenterForMedicalGeneticsGhent-nf-cmgg-germline-methods-description":

diff --git a/assets/samplesheet_only_gvcf.csv b/assets/samplesheet_only_gvcf.csv
@@ -0,0 +1,3 @@
+sample,family,gvcf,tbi
+DNA049572,Proband_12345,https://raw.githubusercontent.com/CenterForMedicalGeneticsGhent/nf-cmgg-test-datasets/main/data/genomics/homo_sapiens/illumina/vcf/test.g.vcf.gz,
+NA24385D2_NVQ_034,Proband_12345,https://raw.githubusercontent.com/CenterForMedicalGeneticsGhent/nf-cmgg-test-datasets/main/data/genomics/homo_sapiens/illumina/vcf/test2.g.vcf.gz,https://raw.githubusercontent.com/CenterForMedicalGeneticsGhent/nf-cmgg-test-datasets/main/data/genomics/homo_sapiens/illumina/vcf/test2.g.vcf.gz.tbi
diff --git a/assets/schema_input.json b/assets/schema_input.json
@@ -3,61 +3,64 @@
     "$id": "https://raw.githubusercontent.com/CenterForMedicalGeneticsGhent/nf-cmgg-germline/master/assets/schema_input.json",
     "title": "Samplesheet validation schema",
     "description": "Schema for the samplesheet used in this pipeline",
-    "type": "object",
-    "properties": {
-        "sample": {
-            "type": "string",
-            "meta": ["id", "sample"]
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "sample": {
+                "type": "string",
+                "meta": ["id", "sample"]
+            },
+            "family": {
+                "type": "string",
+                "meta": ["family"]
+            },
+            "cram": {
+                "type": "string",
+                "pattern": "^\\S+\\.cram$",
+                "format": "file-path"
+            },
+            "crai": {
+                "type": "string",
+                "pattern": "^\\S+\\.crai$",
+                "format": "file-path"
+            },
+            "gvcf": {
+                "type": "string",
+                "pattern": "^\\S+vcf(.gz)?$",
+                "format": "file-path"
+            },
+            "tbi": {
+                "type": "string",
+                "pattern": "^\\S+\\.tbi$",
+                "format": "file-path"
+            },
+            "roi": {
+                "type": "string",
+                "pattern": "^\\S+\\.bed(\\.gz)?$",
+                "format": "file-path"
+            },
+            "ped": {
+                "type": "string",
+                "pattern": "^\\S+\\.ped$",
+                "format": "file-path"
+            },
+            "truth_vcf": {
+                "type": "string",
+                "pattern": "^\\S+\\.vcf\\.gz$",
+                "format": "file-path"
+            },
+            "truth_tbi": {
+                "type": "string",
+                "pattern": "^\\S+\\.tbi$",
+                "format": "file-path"
+            },
+            "truth_bed": {
+                "type": "string",
+                "pattern": "^\\S+\\.bed$",
+                "format": "file-path"
+            }
         },
-        "family": {
-            "type": "string",
-            "meta": ["family"]
-        },
-        "cram": {
-            "type": "string",
-            "pattern": "^\\S+\\.cram$",
-            "format": "file-path"
-        },
-        "crai": {
-            "type": "string",
-            "pattern": "^\\S+\\.crai$",
-            "format": "file-path"
-        },
-        "gvcf": {
-            "type": "string",
-            "pattern": "^\\S+vcf(.gz)?$",
-            "format": "file-path"
-        },
-        "tbi": {
-            "type": "string",
-            "pattern": "^\\S+\\.tbi$",
-            "format": "file-path"
-        },
-        "roi": {
-            "type": "string",
-            "pattern": "^\\S+\\.bed(\\.gz)?$",
-            "format": "file-path"
-        },
-        "ped": {
-            "type": "string",
-            "pattern": "^\\S+\\.ped$",
-            "format": "file-path"
-        },
-        "truth_vcf": {
-            "type": "string",
-            "pattern": "^\\S+\\.vcf\\.gz$",
-            "format": "file-path"
-        },
-        "truth_tbi": {
-            "type": "string",
-            "pattern": "^\\S+\\.tbi$",
-            "format": "file-path"
-        },
-        "truth_bed": {
-            "type": "string",
-            "pattern": "^\\S+\\.bed$",
-            "format": "file-path"
-        }
-    },
-    "required": ["sample", "cram"]
+        "required": ["sample"]
+    }
 }
diff --git a/assets/slackreport.json b/assets/slackreport.json
@@ -3,7 +3,7 @@
         {
             "fallback": "Plain-text summary of the attachment.",
             "color": "<% if (success) { %>good<% } else { %>danger<%} %>",
-            "author_name": "sanger-tol/readmapping v${version} - ${runName}",
+            "author_name": "CenterForMedicalGeneticsGhent/nf-cmgg-germline v${version} - ${runName}",
             "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico",
             "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>",
             "fields": [