From c0326c3d44caf31f1c59673059e70962b1ff2d2e Mon Sep 17 00:00:00 2001 From: aawdeh Date: Thu, 14 Nov 2024 12:49:59 -0500 Subject: [PATCH 1/5] Add filtered option as output to mergestaroutput task --- tasks/skylab/StarAlign.wdl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tasks/skylab/StarAlign.wdl b/tasks/skylab/StarAlign.wdl index ffceb7ce17..a1cae83685 100644 --- a/tasks/skylab/StarAlign.wdl +++ b/tasks/skylab/StarAlign.wdl @@ -482,7 +482,10 @@ task MergeStarOutput { } command <<< - set -e + set -euo pipefail + set -x + + # declare arrays for the files declare -a barcodes_files=(~{sep=' ' barcodes}) declare -a features_files=(~{sep=' ' features}) declare -a matrix_files=(~{sep=' ' matrix}) @@ -503,10 +506,8 @@ task MergeStarOutput { cp ~{input_id}.uniform.mtx ./matrix/matrix.mtx cp ~{barcodes_single} ./matrix/barcodes.tsv cp ~{features_single} ./matrix/features.tsv - tar -zcvf ~{input_id}.mtx_files.tar ./matrix/* - # Running star for combined cell matrix # outputs will be called outputbarcodes.tsv. outputmatrix.mtx, and outputfeatures.tsv STAR --runMode soloCellFiltering ./matrix ./output --soloCellFilter EmptyDrops_CR @@ -515,8 +516,6 @@ task MergeStarOutput { echo "listing files" ls - - if [ -f "${cell_reads_files[0]}" ]; then # Destination file for cell reads @@ -594,13 +593,18 @@ task MergeStarOutput { echo "No text files found in the folder." fi - # # create the compressed raw count matrix with the counts, gene names and the barcodes python3 /scripts/scripts/create-merged-npz-output.py \ --barcodes ${barcodes_files[@]} \ --features ${features_files[@]} \ --matrix ${matrix_files[@]} \ --input_id ~{input_id} + + # tar up filtered matrix outputbarcodes.tsv, outputfeatures.tsv, outputmatrix.mtx + ls + echo "Tarring up filtered matrix files" + tar -cvf ~{input_id}_filtered_mtx_files.tar outputbarcodes.tsv outputfeatures.tsv outputmatrix.mtx + echo "Done" >>> runtime { @@ -619,6 +623,7 @@ task MergeStarOutput { File? cell_reads_out = "~{input_id}.star_metrics.tar" File? library_metrics="~{input_id}_library_metrics.csv" File? mtx_files ="~{input_id}.mtx_files.tar" + File? filtered_mtx_files = "~{input_id}_filtered_mtx_files.tar" File? outputbarcodes = "outputbarcodes.tsv" } } From 534aa96d751ef17a5b71018bda17a731d80da4f8 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Wed, 27 Nov 2024 09:38:26 -0500 Subject: [PATCH 2/5] updated changelogs/version numbers --- pipelines/skylab/multiome/Multiome.changelog.md | 5 +++++ pipelines/skylab/multiome/Multiome.wdl | 2 +- pipelines/skylab/optimus/Optimus.changelog.md | 5 +++++ pipelines/skylab/optimus/Optimus.wdl | 2 +- pipelines/skylab/paired_tag/PairedTag.changelog.md | 5 +++++ pipelines/skylab/paired_tag/PairedTag.wdl | 2 +- pipelines/skylab/slideseq/SlideSeq.changelog.md | 5 +++++ pipelines/skylab/slideseq/SlideSeq.wdl | 2 +- .../MultiSampleSmartSeq2SingleNucleus.changelog.md | 5 +++++ .../MultiSampleSmartSeq2SingleNucleus.wdl | 2 +- 10 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md index 4a05f926be..612bcdf2d4 100644 --- a/pipelines/skylab/multiome/Multiome.changelog.md +++ b/pipelines/skylab/multiome/Multiome.changelog.md @@ -1,3 +1,8 @@ +# 5.9.3 +2024-11-27 (Date of Last Commit) + +* Added filtered_mtx_files as an intermediate output to MergeStarOutput task; this does not affect the outputs of the pipeline + # 5.9.2 2024-11-15 (Date of Last Commit) diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index 438c49d264..93b88997a2 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -9,7 +9,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils workflow Multiome { - String pipeline_version = "5.9.2" + String pipeline_version = "5.9.3" input { String cloud_provider diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md index d05e53bee5..c3fb8baedd 100644 --- a/pipelines/skylab/optimus/Optimus.changelog.md +++ b/pipelines/skylab/optimus/Optimus.changelog.md @@ -1,3 +1,8 @@ +# 7.8.4 +2024-11-27 (Date of Last Commit) + +* Added filtered_mtx_files as an intermediate output to MergeStarOutput task; this does not affect the outputs of the pipeline + # 7.8.3 2024-11-15 (Date of Last Commit) diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl index 41a71d3cb6..8254ec47d6 100644 --- a/pipelines/skylab/optimus/Optimus.wdl +++ b/pipelines/skylab/optimus/Optimus.wdl @@ -71,7 +71,7 @@ workflow Optimus { # version of this pipeline - String pipeline_version = "7.8.3" + String pipeline_version = "7.8.4" # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md index e411e0f679..5fcb32a30c 100644 --- a/pipelines/skylab/paired_tag/PairedTag.changelog.md +++ b/pipelines/skylab/paired_tag/PairedTag.changelog.md @@ -1,3 +1,8 @@ +# 1.8.4 +2024-11-27 (Date of Last Commit) + +* Added filtered_mtx_files as an intermediate output to MergeStarOutput task; this does not affect the outputs of the pipeline + # 1.8.3 2024-11-15 (Date of Last Commit) diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl index 233dafc685..a15b67a21f 100644 --- a/pipelines/skylab/paired_tag/PairedTag.wdl +++ b/pipelines/skylab/paired_tag/PairedTag.wdl @@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils workflow PairedTag { - String pipeline_version = "1.8.3" + String pipeline_version = "1.8.4" input { diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md index 54c9fb6890..9d234b6ac5 100644 --- a/pipelines/skylab/slideseq/SlideSeq.changelog.md +++ b/pipelines/skylab/slideseq/SlideSeq.changelog.md @@ -1,3 +1,8 @@ +# 3.4.7 +2024-11-27 (Date of Last Commit) + +* Added filtered_mtx_files as an intermediate output to MergeStarOutput task; this does not affect the outputs of the pipeline + # 3.4.6 2024-11-15 (Date of Last Commit) diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl index 7c69d79232..c248023e34 100644 --- a/pipelines/skylab/slideseq/SlideSeq.wdl +++ b/pipelines/skylab/slideseq/SlideSeq.wdl @@ -25,7 +25,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils workflow SlideSeq { - String pipeline_version = "3.4.6" + String pipeline_version = "3.4.7" input { Array[File] r1_fastq diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md index 85dc657364..5a7f9b614f 100644 --- a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md +++ b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md @@ -1,3 +1,8 @@ +# 2.0.6 +2024-11-27 (Date of Last Commit) + +* Added filtered_mtx_files as an intermediate output to MergeStarOutput task; this does not affect the outputs of the pipeline + # 2.0.5 2024-11-15 (Date of Last Commit) diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl index 871090e2be..5616b2546c 100644 --- a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl +++ b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl @@ -57,7 +57,7 @@ workflow MultiSampleSmartSeq2SingleNucleus { } # Version of this pipeline - String pipeline_version = "2.0.5" + String pipeline_version = "2.0.6" if (false) { String? none = "None" From 0aba652085449dc3d1515298b7cb5cb1cf221677 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 27 Nov 2024 14:38:52 +0000 Subject: [PATCH 3/5] Updated pipeline_versions.txt with all pipeline version information --- pipeline_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipeline_versions.txt b/pipeline_versions.txt index 9964940111..9b959a7bcf 100644 --- a/pipeline_versions.txt +++ b/pipeline_versions.txt @@ -30,11 +30,11 @@ ExomeReprocessing 3.3.3 2024-11-04 BuildIndices 3.0.0 2023-12-06 scATAC 1.3.2 2023-08-03 snm3C 4.0.4 2024-08-06 -Multiome 5.9.2 2024-11-15 -PairedTag 1.8.3 2024-11-15 +Multiome 5.9.3 2024-11-27 +PairedTag 1.8.4 2024-11-27 MultiSampleSmartSeq2 2.2.22 2024-09-11 -MultiSampleSmartSeq2SingleNucleus 2.0.5 2024-11-15 -Optimus 7.8.3 2024-11-15 +MultiSampleSmartSeq2SingleNucleus 2.0.6 2024-11-27 +Optimus 7.8.4 2024-11-27 atac 2.5.2 2024-11-12 SmartSeq2SingleSample 5.1.21 2024-09-11 -SlideSeq 3.4.6 2024-11-15 +SlideSeq 3.4.7 2024-11-27 From 249228ef5eaeba49f6311b4f3923807a712cd0f8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 9 Jan 2025 16:24:30 +0000 Subject: [PATCH 4/5] Updated pipeline_versions.txt with all pipeline version information --- pipeline_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipeline_versions.txt b/pipeline_versions.txt index b24084e5fe..a75b433425 100644 --- a/pipeline_versions.txt +++ b/pipeline_versions.txt @@ -30,11 +30,11 @@ ExomeReprocessing 3.3.3 2024-11-04 BuildIndices 3.1.0 2024-11-26 scATAC 1.3.2 2023-08-03 snm3C 4.0.4 2024-08-06 -Multiome 5.9.4 2024-12-05 -PairedTag 1.9.0 2024-12-05 +Multiome 5.9.4 2025-01-09 +PairedTag 1.9.0 2025-01-09 MultiSampleSmartSeq2 2.2.22 2024-09-11 -MultiSampleSmartSeq2SingleNucleus 2.0.6 2024-11-15 -Optimus 7.9.0 2024-12-05 +MultiSampleSmartSeq2SingleNucleus 2.0.7 2025-01-09 +Optimus 7.9.0 2025-01-09 atac 2.5.3 2024-11-22 SmartSeq2SingleSample 5.1.21 2024-09-11 -SlideSeq 3.4.7 2024-12-3 +SlideSeq 3.4.8 2024-01-09 From 2e4cbe9dfa839e708e9811126b420848d0591dd3 Mon Sep 17 00:00:00 2001 From: npetrill Date: Thu, 9 Jan 2025 11:25:24 -0500 Subject: [PATCH 5/5] updating changelogs --- pipelines/skylab/slideseq/SlideSeq.wdl | 2 +- .../MultiSampleSmartSeq2SingleNucleus.wdl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl index e43f07979c..a19397b1d7 100644 --- a/pipelines/skylab/slideseq/SlideSeq.wdl +++ b/pipelines/skylab/slideseq/SlideSeq.wdl @@ -25,7 +25,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils workflow SlideSeq { - String pipeline_version = "3.4.7" + String pipeline_version = "3.4.8" input { Array[File] r1_fastq diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl index 5616b2546c..ded336b418 100644 --- a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl +++ b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl @@ -57,7 +57,7 @@ workflow MultiSampleSmartSeq2SingleNucleus { } # Version of this pipeline - String pipeline_version = "2.0.6" + String pipeline_version = "2.0.7" if (false) { String? none = "None"