working outputting of logs

nf-core · Feb 9, 2024 · f67ba4b · f67ba4b
1 parent a5a9383
commit f67ba4b
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 75 deletions.
diff --git a/CITATION.cff b/CITATION.cff
@@ -25,28 +25,28 @@ doi: 10.7717/peerj.10947
 date-released: 2022-08-02
 url: https://github.com/nf-core/eager
 prefered-citation:
-    type: article
-    authors:
-        - family-names: Fellows Yates
-          given-names: James A.
-        - family-names: Lamnidis
-          given-names: Thiseas C.
-        - family-names: Borry
-          given-names: Maxime
-        - family-names: Andrades Valtueña
-          given-names: Aida
-        - family-names: Fagernãs
-          given-names: Zandra
-        - family-names: Clayton
-          given-names: Stephen
-        - family-names: Garcia
-          given-names: Maxime U.
-        - family-names: Neukamm
-          given-names: Judith
-        - family-names: Peltzer
-          given-names: Alexander
-    doi: 10.7717/peerj.10947
-    start: e10947
-    title: "Reproducible, portable, and efficient ancient genome reconstruction with nf-core/eager"
-    year: 2021
-    url: https://dx.doi.org/10.1038/10.7717/peerj.10947
+  type: article
+  authors:
+    - family-names: Fellows Yates
+      given-names: James A.
+    - family-names: Lamnidis
+      given-names: Thiseas C.
+    - family-names: Borry
+      given-names: Maxime
+    - family-names: Andrades Valtueña
+      given-names: Aida
+    - family-names: Fagernãs
+      given-names: Zandra
+    - family-names: Clayton
+      given-names: Stephen
+    - family-names: Garcia
+      given-names: Maxime U.
+    - family-names: Neukamm
+      given-names: Judith
+    - family-names: Peltzer
+      given-names: Alexander
+  doi: 10.7717/peerj.10947
+  start: e10947
+  title: "Reproducible, portable, and efficient ancient genome reconstruction with nf-core/eager"
+  year: 2021
+  url: https://dx.doi.org/10.1038/10.7717/peerj.10947
diff --git a/assets/email_template.html b/assets/email_template.html
@@ -1,53 +1,113 @@
 <html>
-<head>
-  <meta charset="utf-8">
-  <meta http-equiv="X-UA-Compatible" content="IE=edge">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
 
-  <meta name="description" content="nf-core/eager: A fully reproducible and state-of-the-art ancient DNA analysis pipeline">
-  <title>nf-core/eager Pipeline Report</title>
-</head>
-<body>
-<div style="font-family: Helvetica, Arial, sans-serif; padding: 30px; max-width: 800px; margin: 0 auto;">
+    <meta
+      name="description"
+      content="nf-core/eager: A fully reproducible and state-of-the-art ancient DNA analysis pipeline"
+    />
+    <title>nf-core/eager Pipeline Report</title>
+  </head>
+  <body>
+    <div style="font-family: Helvetica, Arial, sans-serif; padding: 30px; max-width: 800px; margin: 0 auto">
+      <img src="cid:nfcorepipelinelogo" />
 
-<img src="cid:nfcorepipelinelogo">
+      <h1>nf-core/eager v${version}</h1>
+      <h2>Run Name: $runName</h2>
 
-<h1>nf-core/eager v${version}</h1>
-<h2>Run Name: $runName</h2>
-
-<% if (!success){
-    out << """
-    <div style="color: #a94442; background-color: #f2dede; border-color: #ebccd1; padding: 15px; margin-bottom: 20px; border: 1px solid transparent; border-radius: 4px;">
-        <h4 style="margin-top:0; color: inherit;">nf-core/eager execution completed unsuccessfully!</h4>
+      <% if (!success){ out << """
+      <div
+        style="
+          color: #a94442;
+          background-color: #f2dede;
+          border-color: #ebccd1;
+          padding: 15px;
+          margin-bottom: 20px;
+          border: 1px solid transparent;
+          border-radius: 4px;
+        "
+      >
+        <h4 style="margin-top: 0; color: inherit">nf-core/eager execution completed unsuccessfully!</h4>
         <p>The exit status of the task that caused the workflow execution to fail was: <code>$exitStatus</code>.</p>
         <p>The full error message was:</p>
-        <pre style="white-space: pre-wrap; overflow: visible; margin-bottom: 0;">${errorReport}</pre>
-    </div>
-    """
-} else {
-    out << """
-    <div style="color: #3c763d; background-color: #dff0d8; border-color: #d6e9c6; padding: 15px; margin-bottom: 20px; border: 1px solid transparent; border-radius: 4px;">
+        <pre style="white-space: pre-wrap; overflow: visible; margin-bottom: 0">${errorReport}</pre>
+      </div>
+      """ } else { out << """
+      <div
+        style="
+          color: #3c763d;
+          background-color: #dff0d8;
+          border-color: #d6e9c6;
+          padding: 15px;
+          margin-bottom: 20px;
+          border: 1px solid transparent;
+          border-radius: 4px;
+        "
+      >
         nf-core/eager execution completed successfully!
-    </div>
-    """
-}
-%>
+      </div>
+      """ } %>
 
-<p>The workflow was completed at <strong>$dateComplete</strong> (duration: <strong>$duration</strong>)</p>
-<p>The command used to launch the workflow was as follows:</p>
-<pre style="white-space: pre-wrap; overflow: visible; background-color: #ededed; padding: 15px; border-radius: 4px; margin-bottom:30px;">$commandLine</pre>
+      <p>The workflow was completed at <strong>$dateComplete</strong> (duration: <strong>$duration</strong>)</p>
+      <p>The command used to launch the workflow was as follows:</p>
+      <pre
+        style="
+          white-space: pre-wrap;
+          overflow: visible;
+          background-color: #ededed;
+          padding: 15px;
+          border-radius: 4px;
+          margin-bottom: 30px;
+        "
+      >
+$commandLine</pre
+      >
 
-<h3>Pipeline Configuration:</h3>
-<table style="width:100%; max-width:100%; border-spacing: 0; border-collapse: collapse; border:0; margin-bottom: 30px;">
-    <tbody style="border-bottom: 1px solid #ddd;">
-        <% out << summary.collect{ k,v -> "<tr><th style='text-align:left; padding: 8px 0; line-height: 1.42857143; vertical-align: top; border-top: 1px solid #ddd;'>$k</th><td style='text-align:left; padding: 8px; line-height: 1.42857143; vertical-align: top; border-top: 1px solid #ddd;'><pre style='white-space: pre-wrap; overflow: visible;'>$v</pre></td></tr>" }.join("\n") %>
-    </tbody>
-</table>
+      <h3>Pipeline Configuration:</h3>
+      <table
+        style="
+          width: 100%;
+          max-width: 100%;
+          border-spacing: 0;
+          border-collapse: collapse;
+          border: 0;
+          margin-bottom: 30px;
+        "
+      >
+        <tbody style="border-bottom: 1px solid #ddd">
+          <% out << summary.collect{ k,v -> "
+          <tr>
+            <th
+              style="
+                text-align: left;
+                padding: 8px 0;
+                line-height: 1.42857143;
+                vertical-align: top;
+                border-top: 1px solid #ddd;
+              "
+            >
+              $k
+            </th>
+            <td
+              style="
+                text-align: left;
+                padding: 8px;
+                line-height: 1.42857143;
+                vertical-align: top;
+                border-top: 1px solid #ddd;
+              "
+            >
+              <pre style="white-space: pre-wrap; overflow: visible">$v</pre>
+            </td>
+          </tr>
+          " }.join("\n") %>
+        </tbody>
+      </table>
 
-<p>nf-core/eager</p>
-<p><a href="https://github.com/nf-core/eager">https://github.com/nf-core/eager</a></p>
-
-</div>
-
-</body>
+      <p>nf-core/eager</p>
+      <p><a href="https://github.com/nf-core/eager">https://github.com/nf-core/eager</a></p>
+    </div>
+  </body>
 </html>
diff --git a/conf/modules.config b/conf/modules.config
@@ -845,6 +845,7 @@ process {
             mode: params.publish_dir_mode,
             pattern: '*.{rma6,log,sam.gz}'
         ]
+        ext.prefix = { "${meta.label}_${meta.id}-run" }
     }
 
     withName: KRAKEN2_KRAKEN2 {

diff --git a/docs/development/manual_tests.md b/docs/development/manual_tests.md
@@ -702,7 +702,12 @@ nextflow run main.nf -profile test,docker \
 
 # for generating test data
 mkdir testing && cd testing
+wget https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta
 git clone https://github.com/rhuebler/HOPS.git
+cd HOPS/Test_Data/Test_Database
+unzip table0.db.zip
+unzip table0.idx.zip
+cd ../../..
 mkdir test_data && cd test_data
 curl -L ftp://ftp.sra.ebi.ac.uk/vol1/fastq/ERR895/000/ERR8958750/ERR8958750_1.fastq.gz -o ERR8958750_1.fastq.gz
 curl -L ftp://ftp.sra.ebi.ac.uk/vol1/fastq/ERR895/000/ERR8958750/ERR8958750_2.fastq.gz -o ERR8958750_2.fastq.gz
@@ -716,6 +721,7 @@ curl -L ftp://ftp.sra.ebi.ac.uk/vol1/fastq/ERR895/004/ERR8958754/ERR8958754_1.fa
 curl -L ftp://ftp.sra.ebi.ac.uk/vol1/fastq/ERR895/004/ERR8958754/ERR8958754_2.fastq.gz -o ERR8958754_2.fastq.gz
 ls -1 | while read file; do
 zcat $file | head -n 4000 > ${file}_reduced.fastq
+gzip ${file}_reduced.fastq
 done
 cd ..
 

diff --git a/docs/output.md b/docs/output.md
@@ -354,7 +354,7 @@ The saved files are the _good_ files, passing the `dust` or `entropy` filter tre
 - `metagenomics_screening/profiling/malt/`
   - `<sample_id>.rma6`: binary file containing all alignments and taxonomic information of hits that can be loaded into the [MEGAN6](https://uni-tuebingen.de/fakultaeten/mathematisch-naturwissenschaftliche-fakultaet/fachbereiche/informatik/lehrstuehle/algorithms-in-bioinformatics/software/megan6/) interactive viewer
   - `<sample_id>.blastn.sam`: sparse SAM file containing alignments of each hit (if `--metagenomics_malt_savereads`)
-  - `*.log`: LOG file containing runtime log of MALT
+  - `*.log`: LOG file containing runtime log of MALT. NOTE: If you are running parallel malt runs with `--metagenomics_malt_group_size` set above 0, your log files will be labelled with the name of _one_ of the input files run for each of the parallel executions.
 
 </details>
 

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -812,7 +812,7 @@
                     "default": 0,
                     "description": "Define how many fastq files should be submitted in the same malt run. Default value of 0 sends all files at once.",
                     "fa_icon": "fas fa-barcode",
-                    "help_text": "Very large fastq files or many fastq files run through MALT at the same time can lead to excessively long runtimes. This parameter allows for parallelization of MALT runs. Please note, MALT is resource heavy and setting this value above the default (0) will spawn at minimum N/metagenomics_malt_group_size jobs where N is the number of samples. Please only use this if it is necessary to avoid runtime limits on your HPC cluster."
+                    "help_text": "Very many (large) fastq files run through MALT at the same time can lead to excessively long runtimes. This parameter allows for parallelization of MALT runs. Please note, MALT is resource heavy and setting this value above the default (0) will spawn at minimum N/metagenomics_malt_group_size jobs where N is the number of samples. Please only use this if it is necessary to avoid runtime limits on your HPC cluster since the overhead of loading a database is high."
                 },
                 "metagenomics_run_postprocessing": {
                     "type": "boolean",

diff --git a/subworkflows/local/metagenomics_postprocessing.nf b/subworkflows/local/metagenomics_postprocessing.nf
@@ -21,7 +21,6 @@ workflow METAGENOMICS_POSTPROCESSING {
             meta, rma, rma_collected ->
                 rma_collected
             }
-        ch_maltextract_input.view()
 
         tax_list = Channel.fromPath(params.metagenomics_maltextract_taxonlist)
         ncbi_dir = Channel.fromPath(params.metagenomics_maltextract_ncbidir)

diff --git a/subworkflows/local/metagenomics_profiling.nf b/subworkflows/local/metagenomics_profiling.nf
@@ -37,21 +37,28 @@ workflow METAGENOMICS_PROFILING {
         // unnecessary. Set as database name to prevent `null` job ID and prefix.
 
         if ( params.metagenomics_malt_group_size > 0 ) {
+            ch_labels_for_malt_tmp = reads
+                .map { meta, reads -> meta }
+                .collate(params.metagenomics_malt_group_size)
+                .map(meta -> meta.first().library_id )
+
             ch_input_for_malt_tmp =  reads
                 .map { meta, reads -> reads }
                 .collate( params.metagenomics_malt_group_size ) //collate into bins of defined lengths
                 .map{
                     reads ->
                     // add new meta with db-name as id
-                    [[id: file(params.metagenomics_profiling_database).getBaseName() ], reads]
+                    [[label: file(params.metagenomics_profiling_database).getBaseName() ], reads]
                 }
-                .combine(database) //combine with database
+                .combine(database)
+                .merge(ch_labels_for_malt_tmp) //combine with database
                 .multiMap{
                     // and split apart again
-                    meta, reads, database ->
-                        reads: [meta, reads]
+                    meta, reads, database, ids ->
+                        reads: [meta + ['id':ids], reads]
                         database: database
                 }
+
             ch_input_for_malt = ch_input_for_malt_tmp.reads
             database = ch_input_for_malt_tmp.database
         }
@@ -63,7 +70,7 @@ workflow METAGENOMICS_PROFILING {
                 .map{
                     // make sure id is db_name for publishing purposes.
                     reads ->
-                    [[id: file(params.metagenomics_profiling_database).getBaseName() ], reads]
+                    [[label: file(params.metagenomics_profiling_database).getBaseName(), id: 'all' ], reads]
                 }
         }