From 1c7c4b9fc35c5699b1a3126b377413354d5133d3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 17 Oct 2023 14:24:28 +0200 Subject: [PATCH 01/84] build: Add wave --- tests/config/nextflow.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c24e252ec57..40f7ef8c862 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,6 +14,7 @@ process { if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true + wave.enabled = true } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { @@ -28,6 +29,7 @@ if ("$PROFILE" == "singularity") { docker.userEmulation = false docker.fixOwnership = true docker.runOptions = '--platform=linux/amd64' + wave.enabled = true } docker.registry = 'quay.io' From 112c9e6eba2910a0a26680bf139a9c10bffa2c97 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 17 Oct 2023 14:25:56 +0200 Subject: [PATCH 02/84] build: Set strategy to dockerfile, conda then container --- tests/config/nextflow.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 40f7ef8c862..68c18b4a3e4 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -15,6 +15,7 @@ if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true wave.enabled = true + wave.strategy = ['dockerfile', 'conda', 'container'] } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { @@ -30,6 +31,7 @@ if ("$PROFILE" == "singularity") { docker.fixOwnership = true docker.runOptions = '--platform=linux/amd64' wave.enabled = true + wave.strategy = ['dockerfile', 'conda', 'container'] } docker.registry = 'quay.io' From 9d61b64083c7ad40346ebbb0c691121810042d1e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 17 Oct 2023 17:19:21 +0200 Subject: [PATCH 03/84] refactor: Remove container --- modules/nf-core/bowtie/align/main.nf | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 29e9cd53361..4ee978de723 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,9 +3,6 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' : - 'biocontainers/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' }" input: tuple val(meta), path(reads) From cf3aa942038739beed9b1b216e4dc9993aa43e97 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 17 Oct 2023 17:53:49 +0200 Subject: [PATCH 04/84] build: Add a repo to push to --- tests/config/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 68c18b4a3e4..1e646eda5f2 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -32,6 +32,7 @@ if ("$PROFILE" == "singularity") { docker.runOptions = '--platform=linux/amd64' wave.enabled = true wave.strategy = ['dockerfile', 'conda', 'container'] + wave.build.repository = 'docker.io/emiller88/modules' } docker.registry = 'quay.io' From 73d6d6f2ec41c5412cf752b446082c760b4ad58a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:10:49 -0500 Subject: [PATCH 05/84] ci(wave): Add wave build https://github.com/nodejs/docker-node/blob/3c4fa6daf06a4786d202f2f610351837806a0380/.github/workflows/build-test.yml#L29 --- .github/workflows/wave.yml | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/wave.yml diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml new file mode 100644 index 00000000000..e38d4d9beec --- /dev/null +++ b/.github/workflows/wave.yml @@ -0,0 +1,60 @@ +name: Build containers +# When environment.yml is changed +on: + pull_request: + paths: + - "**/Dockerfile" + - "**/environment.yml" + +env: + WAVE_VER: "1.0.0" + +jobs: + gen-matrix: + name: generate-matrix + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Calculate file differences + id: diff + uses: tj-actions/changed-files@v35 + with: + json: true + json_raw_format: true + - name: Debug + run: echo ${{ steps.diff.outputs.modified_files }} + + outputs: + matrix: ${{ steps.diff.outputs.modified_files }} + + build: + if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} + needs: gen-matrix + name: build + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} + steps: + # Use the @nf-core-bot token to check out so we can push later + - uses: actions/checkout@v3 + with: + token: ${{ secrets.nf_core_bot_auth_token }} + + - name: Install wave-cli + run: | + wget -qO- https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 + sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + + - name: Build container + run: | + wave --conda-file "${{ matrix.file }}" + + # - name: Build Singularity + # run: | + # container=$(wave --singularity --conda-package bamtools=2.5.2 --build-repo docker.io/user/repo --freeze --await) + # singularity exec $container bamtools --version From 83f1e30fffb44301fcbc17b4334bd6b641a95993 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:12:38 -0500 Subject: [PATCH 06/84] ci(wave): Switch to all_changed_files --- .github/workflows/wave.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index e38d4d9beec..cae17ecf0ce 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -25,10 +25,10 @@ jobs: json: true json_raw_format: true - name: Debug - run: echo ${{ steps.diff.outputs.modified_files }} + run: echo ${{ steps.diff.outputs.all_changed_files }} outputs: - matrix: ${{ steps.diff.outputs.modified_files }} + matrix: ${{ steps.diff.outputs.all_changed_files }} build: if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} From 95d00697c27ec2130dde70ff82774fab1895f220 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:13:29 -0500 Subject: [PATCH 07/84] ci(wave): Only look for envronment.ymls --- .github/workflows/wave.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index cae17ecf0ce..01cf997e9d3 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -24,6 +24,9 @@ jobs: with: json: true json_raw_format: true + # TODO Add Dockerfiles + files: | + modules/**/environment.yml - name: Debug run: echo ${{ steps.diff.outputs.all_changed_files }} From 649214d2ec0ed3c59a444b40e37856e74577aa84 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:34:40 -0500 Subject: [PATCH 08/84] dummy: Change env --- modules/nf-core/bowtie/align/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/bowtie/align/environment.yml b/modules/nf-core/bowtie/align/environment.yml index 2617e6f0adc..fb5d7c1ef12 100644 --- a/modules/nf-core/bowtie/align/environment.yml +++ b/modules/nf-core/bowtie/align/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::bowtie=1.3.0 + - bioconda::samtools=1.16.1 From a4f4611c976477327a623b069e7ad863e1d2a77d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:35:54 -0500 Subject: [PATCH 09/84] ci(wave): Remove raw format --- .github/workflows/wave.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 01cf997e9d3..23866bfc34c 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -23,7 +23,6 @@ jobs: uses: tj-actions/changed-files@v35 with: json: true - json_raw_format: true # TODO Add Dockerfiles files: | modules/**/environment.yml From aa02e1bf78682377591569827b37b7a85222894a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:49:34 -0500 Subject: [PATCH 10/84] ci(wave): Try a bunch of different things at once --- .github/workflows/wave.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 23866bfc34c..eadf9a02eda 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -33,7 +33,7 @@ jobs: matrix: ${{ steps.diff.outputs.all_changed_files }} build: - if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} + # if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} needs: gen-matrix name: build runs-on: ubuntu-latest @@ -44,8 +44,6 @@ jobs: steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 - with: - token: ${{ secrets.nf_core_bot_auth_token }} - name: Install wave-cli run: | @@ -54,7 +52,7 @@ jobs: - name: Build container run: | - wave --conda-file "${{ matrix.file }}" + wave --conda-file "${{ matrix }}" # - name: Build Singularity # run: | From b37919c74d6249b0b1b6275a6a2a495a5e33163f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 15:57:47 -0500 Subject: [PATCH 11/84] ci(wave): Remove redundant fromJson and wrap in an array --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index eadf9a02eda..7b7848a027f 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -40,7 +40,7 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - matrix: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} + matrix: "${{ needs.gen-matrix.outputs.matrix }}" steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 From 42e9f9e5d72e3b9a65401d50170a595b451aea8a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:08:39 -0500 Subject: [PATCH 12/84] ci(wave): I have no idea what I'm doing --- .github/workflows/wave.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 7b7848a027f..cad8f1a3cd3 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -40,7 +40,8 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - matrix: "${{ needs.gen-matrix.outputs.matrix }}" + matrix: + image: "${{ needs.gen-matrix.outputs.matrix }}" steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 @@ -52,7 +53,7 @@ jobs: - name: Build container run: | - wave --conda-file "${{ matrix }}" + wave --conda-file "${{ matrix.image }}" # - name: Build Singularity # run: | From abdbc573108687003945de3ae15c11475ee8b172 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:16:32 -0500 Subject: [PATCH 13/84] ci(wave): Wrap it --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index cad8f1a3cd3..2364fef5a12 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -41,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - image: "${{ needs.gen-matrix.outputs.matrix }}" + image: ["${{ fromJson(needs.gen-matrix.outputs.matrix) }}"] steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 From 06188ddc2b7b572f8f3bfc44c08d3e8435da207a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:20:12 -0500 Subject: [PATCH 14/84] ci(wave): Found an example https://github.com/tj-actions/changed-files/blob/main/.github/workflows/matrix-test.yml --- .github/workflows/wave.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 2364fef5a12..ecedf9c02bd 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -23,14 +23,17 @@ jobs: uses: tj-actions/changed-files@v35 with: json: true + quotepath: false # TODO Add Dockerfiles files: | modules/**/environment.yml - name: Debug run: echo ${{ steps.diff.outputs.all_changed_files }} + - id: set-matrix + run: echo "matrix={\"files\":${{ steps.diff.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" outputs: - matrix: ${{ steps.diff.outputs.all_changed_files }} + matrix: ${{ steps.set-matrix.outputs.matrix }} build: # if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} @@ -40,8 +43,7 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - matrix: - image: ["${{ fromJson(needs.gen-matrix.outputs.matrix) }}"] + matrix: ${{ fromJSON(needs.changed-files.outputs.matrix) }} steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 @@ -53,7 +55,7 @@ jobs: - name: Build container run: | - wave --conda-file "${{ matrix.image }}" + wave --conda-file "${{ matrix.files }}" # - name: Build Singularity # run: | From 72275d6f7a8330ec6b9e9d31695942ee75540e14 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:25:48 -0500 Subject: [PATCH 15/84] ci(wave): Maybe quotes? --- .github/workflows/wave.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index ecedf9c02bd..650523ae80c 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -36,14 +36,14 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} build: - # if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }} + if: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" needs: gen-matrix name: build runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: ${{ fromJSON(needs.changed-files.outputs.matrix) }} + matrix: "${{ fromJson(needs.changed-files.outputs.matrix) }}" steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 From bd0bdb08eb525077653f77d293fa5c9c6de54638 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:26:52 -0500 Subject: [PATCH 16/84] ci(wave): That'll do it --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 650523ae80c..531a6909335 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -43,7 +43,7 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - matrix: "${{ fromJson(needs.changed-files.outputs.matrix) }}" + matrix: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 From 8333d50e5e032208c9ae2a9f5f3e8d6efce32d29 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:50:22 -0500 Subject: [PATCH 17/84] ci(wave): Fix wave install --- .github/workflows/wave.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 531a6909335..dd48bbc4595 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -50,8 +50,9 @@ jobs: - name: Install wave-cli run: | - wget -qO- https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + chmod +x /usr/local/bin/wave - name: Build container run: | From 95ec7e49bba592b6a8e0473f714b355087d4df02 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 16:54:19 -0500 Subject: [PATCH 18/84] ci(wave): Hard code an image --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index dd48bbc4595..b45e1a22b4d 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -56,7 +56,7 @@ jobs: - name: Build container run: | - wave --conda-file "${{ matrix.files }}" + wave --conda-file "${{ matrix.files }}" --freeze --build-repo ghcr.io/nf-core/modules/bowtie # - name: Build Singularity # run: | From f20da9444a166418f89b32e652685b10e9b75516 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 18:30:05 -0500 Subject: [PATCH 19/84] ci(wave): Add secrets --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index b45e1a22b4d..d5e06337d78 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -56,7 +56,7 @@ jobs: - name: Build container run: | - wave --conda-file "${{ matrix.files }}" --freeze --build-repo ghcr.io/nf-core/modules/bowtie + wave --conda-file "${{ matrix.files }}" --freeze --build-repo ghcr.io/nf-core/modules/bowtie --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} # - name: Build Singularity # run: | From 4fe3bbd859cb4cb093681052cbe93f522ee048ab Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 21:18:32 -0500 Subject: [PATCH 20/84] feat: Try a different files structure --- .github/workflows/wave.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index d5e06337d78..fc7446985b2 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -33,17 +33,18 @@ jobs: run: echo "matrix={\"files\":${{ steps.diff.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + files: ${{ steps.diff.outputs.all_changed_files }} build: - if: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" + if: "${{ fromJson(needs.gen-matrix.outputs.files) }}" needs: gen-matrix name: build runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" + matrix: + files: "${{ fromJson(needs.gen-matrix.outputs.files) }}" steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 From 7f84d8aef2d3cb8c2e5e1de9470e31ea1a29349d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 21:35:36 -0500 Subject: [PATCH 21/84] ci(wave): First stab at building singularity images --- .github/workflows/wave.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index fc7446985b2..787029e048e 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -45,6 +45,7 @@ jobs: fail-fast: false matrix: files: "${{ fromJson(needs.gen-matrix.outputs.files) }}" + profile: ["docker", "singularity"] steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 @@ -57,9 +58,11 @@ jobs: - name: Build container run: | - wave --conda-file "${{ matrix.files }}" --freeze --build-repo ghcr.io/nf-core/modules/bowtie --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + wave --conda-file "${{ matrix.files }}" \ + --freeze \ + --build-repo ghcr.io/nf-core/modules/bowtie \ + --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ + ${[ "${{ matrix.profile }}" == "singularity" ]:+--singularity} - # - name: Build Singularity - # run: | - # container=$(wave --singularity --conda-package bamtools=2.5.2 --build-repo docker.io/user/repo --freeze --await) - # singularity exec $container bamtools --version + # TODO Build from Dockerfiles From c80e959788e73b4b7c4b17e65bcc68f99868dcd0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 21:41:02 -0500 Subject: [PATCH 22/84] fixup! feat: Try a different files structure --- .github/workflows/wave.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 787029e048e..4460bc25831 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -29,8 +29,8 @@ jobs: modules/**/environment.yml - name: Debug run: echo ${{ steps.diff.outputs.all_changed_files }} - - id: set-matrix - run: echo "matrix={\"files\":${{ steps.diff.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" + # - id: set-matrix + # run: echo "matrix={\"files\":${{ steps.diff.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" outputs: files: ${{ steps.diff.outputs.all_changed_files }} From 15b163dc8d64c8e9582fb126bbdc67cc025d62b7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 21:52:50 -0500 Subject: [PATCH 23/84] ci(wave): Add profile to matrix --- .github/workflows/wave.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 4460bc25831..a98c29504e9 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -29,23 +29,21 @@ jobs: modules/**/environment.yml - name: Debug run: echo ${{ steps.diff.outputs.all_changed_files }} - # - id: set-matrix - # run: echo "matrix={\"files\":${{ steps.diff.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT" + - id: set-matrix + run: echo "matrix={\"profiles\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" outputs: - files: ${{ steps.diff.outputs.all_changed_files }} + matrix: ${{ steps.set-matrix.outputs.matrix }} build: - if: "${{ fromJson(needs.gen-matrix.outputs.files) }}" + if: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" needs: gen-matrix name: build runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: - files: "${{ fromJson(needs.gen-matrix.outputs.files) }}" - profile: ["docker", "singularity"] + matrix: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" steps: # Use the @nf-core-bot token to check out so we can push later - uses: actions/checkout@v3 From a08373d3c3335d6f420397aca67d0840a9b870a3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 2 Nov 2023 22:35:46 -0500 Subject: [PATCH 24/84] ci(wave): Give up on fancy substitution --- .github/workflows/wave.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index a98c29504e9..db5bb0038c6 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -30,7 +30,7 @@ jobs: - name: Debug run: echo ${{ steps.diff.outputs.all_changed_files }} - id: set-matrix - run: echo "matrix={\"profiles\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" + run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -55,12 +55,22 @@ jobs: chmod +x /usr/local/bin/wave - name: Build container + if: matrix.profile == 'docker' + run: | + wave --conda-file "${{ matrix.files }}" \ + --freeze \ + --build-repo ghcr.io/nf-core/modules/bowtie \ + --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + + - name: Build Singularity + if: matrix.profile == 'singularity' run: | wave --conda-file "${{ matrix.files }}" \ --freeze \ --build-repo ghcr.io/nf-core/modules/bowtie \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ - ${[ "${{ matrix.profile }}" == "singularity" ]:+--singularity} + --singularity # TODO Build from Dockerfiles From 960702fb0506b4c8402f00947495327466ff9769 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 09:36:25 -0500 Subject: [PATCH 25/84] ci(wave): Add await Co-authored-by: ewels --- .github/workflows/wave.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index db5bb0038c6..1836b00a8f8 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -59,6 +59,7 @@ jobs: run: | wave --conda-file "${{ matrix.files }}" \ --freeze \ + --await \ --build-repo ghcr.io/nf-core/modules/bowtie \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -68,6 +69,7 @@ jobs: run: | wave --conda-file "${{ matrix.files }}" \ --freeze \ + --await \ --build-repo ghcr.io/nf-core/modules/bowtie \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ From 9a3f93e9c40870a9df9b346b60cc391042ebee27 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 14:20:26 -0500 Subject: [PATCH 26/84] ci(wave): Switch to quay --- .github/workflows/wave.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 1836b00a8f8..f32d81b0a8d 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -60,7 +60,7 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo ghcr.io/nf-core/modules/bowtie \ + --build-repo quay.io/nf-core/modules/bowtie \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -70,7 +70,7 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo ghcr.io/nf-core/modules/bowtie \ + --build-repo quay.io/nf-core/modules/bowtie \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity From 4c92894a80f817cad75fbc6284b746b9400bfa23 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 17:34:43 -0500 Subject: [PATCH 27/84] test(wave): Add freeze and update build repo --- tests/config/nextflow.config | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 1e646eda5f2..ada29007f55 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -16,6 +16,8 @@ if ("$PROFILE" == "singularity") { singularity.autoMounts = true wave.enabled = true wave.strategy = ['dockerfile', 'conda', 'container'] + wave.freeze = true + wave.build.repository = 'quay.io/nf-core/modules' } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { @@ -32,7 +34,8 @@ if ("$PROFILE" == "singularity") { docker.runOptions = '--platform=linux/amd64' wave.enabled = true wave.strategy = ['dockerfile', 'conda', 'container'] - wave.build.repository = 'docker.io/emiller88/modules' + wave.freeze = true + wave.build.repository = 'quay.io/nf-core/modules' } docker.registry = 'quay.io' From 1cd2efe5f763dd76c3691ae85e76579b9ef79f3a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 17:36:32 -0500 Subject: [PATCH 28/84] refactor(wave): What happens if I add a container? --- modules/nf-core/bowtie/align/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 4ee978de723..62212fffe9a 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,6 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" + container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--82705d624eee2198" input: tuple val(meta), path(reads) From 6386a76fa5842bbfcdd58ccf6b1491eb76da178c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 17:49:00 -0500 Subject: [PATCH 29/84] refactor(wave): Have both bowtie modules use the same env For the sake of demonstration --- modules/nf-core/bowtie/build/environment.yml | 1 + modules/nf-core/bowtie/build/main.nf | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/bowtie/build/environment.yml b/modules/nf-core/bowtie/build/environment.yml index 0907b0f840f..3b6490b87e5 100644 --- a/modules/nf-core/bowtie/build/environment.yml +++ b/modules/nf-core/bowtie/build/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::bowtie=1.3.0 + - bioconda::samtools=1.16.1 diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 05e22fe8c61..fd01185e417 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,9 +3,7 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bowtie:1.3.0--py38hed8969a_1' : - 'biocontainers/bowtie:1.3.0--py38hed8969a_1' }" + container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--82705d624eee2198" input: path fasta From 8aec35dbdda2927a00297015a4859af51cf1c94b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 18:00:40 -0500 Subject: [PATCH 30/84] test: Cut out using wave on tests --- tests/config/nextflow.config | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index ada29007f55..c24e252ec57 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,10 +14,6 @@ process { if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true - wave.enabled = true - wave.strategy = ['dockerfile', 'conda', 'container'] - wave.freeze = true - wave.build.repository = 'quay.io/nf-core/modules' } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { @@ -32,10 +28,6 @@ if ("$PROFILE" == "singularity") { docker.userEmulation = false docker.fixOwnership = true docker.runOptions = '--platform=linux/amd64' - wave.enabled = true - wave.strategy = ['dockerfile', 'conda', 'container'] - wave.freeze = true - wave.build.repository = 'quay.io/nf-core/modules' } docker.registry = 'quay.io' From 5a8fcc9e5269fb0e446348553ca5e112ae54f419 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 3 Nov 2023 18:11:46 -0500 Subject: [PATCH 31/84] refactor: What happens if we use the singularity one? --- modules/nf-core/bowtie/align/main.nf | 2 +- modules/nf-core/bowtie/build/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 62212fffe9a..4c8e1e9503f 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--82705d624eee2198" + container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index fd01185e417..289ec1038ae 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,7 +3,7 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--82705d624eee2198" + container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d" input: path fasta From 9b8ddfd92089d496307a4c50623c1329ed14f157 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 10 Nov 2023 10:20:15 -0600 Subject: [PATCH 32/84] refactor: Keep container directives for offline download https://github.com/seqeralabs/wave/issues/323 --- modules/nf-core/bowtie/align/main.nf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 4c8e1e9503f..0f399bcc6e0 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,9 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d' : + 'nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--82705d624eee2198' }" input: tuple val(meta), path(reads) From 7b12cd7a8b07d8b237fa7b073f5903268ed9c43d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 22 Nov 2023 11:32:26 -0600 Subject: [PATCH 33/84] feat: Try new singularity OCI setting https://github.com/nextflow-io/nextflow/commit/f5362a7b067173a29d684663df22bb48fbbf5659 --- modules/nf-core/bowtie/align/main.nf | 4 +--- tests/config/nextflow.config | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 0f399bcc6e0..4c8e1e9503f 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,9 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d' : - 'nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--82705d624eee2198' }" + container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d" input: tuple val(meta), path(reads) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c24e252ec57..ebe8c18284b 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,6 +14,7 @@ process { if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true + singularity.oci = true } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { From cac8b9f29c3d8d52ced313f77aaf20b06a6c9889 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 22 Nov 2023 12:00:13 -0600 Subject: [PATCH 34/84] build: Update container name Guess #4327 broke that --- modules/nf-core/bowtie/align/main.nf | 2 +- modules/nf-core/bowtie/build/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 4c8e1e9503f..1f880aa521c 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d" + container "nf-core/modules/bowtie:bowtie_align--6c5b9c93546643d8" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 289ec1038ae..b7e7f861c36 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,7 +3,7 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie-1.3.0_samtools-1.16.1--2c2a8138b83d6d1d" + container "nf-core/modules/bowtie:bowtie_build--df26d88a69745299" input: path fasta From 75464b32abcfa393bcb88abba71166895713cde3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 08:24:58 -0600 Subject: [PATCH 35/84] chore: Bump wave-cli version --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index f32d81b0a8d..51c6e8d626d 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -7,7 +7,7 @@ on: - "**/environment.yml" env: - WAVE_VER: "1.0.0" + WAVE_VER: "1.1.0" jobs: gen-matrix: From f588f186b48296b69e7441471d6def4981ec2d6b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 08:45:35 -0600 Subject: [PATCH 36/84] ci: Install runc --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3977dfec5d..66b20b21704 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -448,6 +448,11 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR + # For OCI images + - name: Install runc + if: matrix.profile == 'singularity' + run: sudo apt-get install -y runc + - name: Set up miniconda uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 with: From f4144ac03b912432b7da0c280fea367b461bef1d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 11:12:54 -0600 Subject: [PATCH 37/84] ci: Switch to singularityhub action https://github.com/nextflow-io/nextflow/issues/4543 --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66b20b21704..80e1466c31d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -438,9 +438,11 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Setup apptainer + - name: Install Singularity if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main + uses: singularityhub/install-singularity@main + with: + singularity-version: 4.0.2 - name: Set up Singularity if: matrix.profile == 'singularity' From 8ea536e21de5baf58b36ea1e7a0735b6a3170352 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 11:33:08 -0600 Subject: [PATCH 38/84] ci: Install new singularity manually Why that action trys to build from source, idk. --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80e1466c31d..ec35fe11102 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -440,9 +440,9 @@ jobs: - name: Install Singularity if: matrix.profile == 'singularity' - uses: singularityhub/install-singularity@main - with: - singularity-version: 4.0.2 + run: | + wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb + sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb - name: Set up Singularity if: matrix.profile == 'singularity' From d6437f8968e046f40faa7147115b6ede5b637014 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 11:47:14 -0600 Subject: [PATCH 39/84] ci: Install dependancies for singularity --- .github/workflows/test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec35fe11102..d547bc269e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -442,6 +442,7 @@ jobs: if: matrix.profile == 'singularity' run: | wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb + sudo apt-get install -y runc libfuse2 sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb - name: Set up Singularity @@ -450,11 +451,6 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR - # For OCI images - - name: Install runc - if: matrix.profile == 'singularity' - run: sudo apt-get install -y runc - - name: Set up miniconda uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 with: From 265abdfee955531a18d943174937b28fbda359df Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 11:57:31 -0600 Subject: [PATCH 40/84] ci: runc => crun --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d547bc269e9..6016446af0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -442,7 +442,7 @@ jobs: if: matrix.profile == 'singularity' run: | wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb - sudo apt-get install -y runc libfuse2 + sudo apt-get install -y crun libfuse2 sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb - name: Set up Singularity From 00313f36e2bcf3f6e981160c926bc82c442aaec5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 26 Nov 2023 12:13:31 -0600 Subject: [PATCH 41/84] ci: Fix cgroup error https://blog.misharov.pro/2021-05-16/systemd-github-actions --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6016446af0c..9ef8f5fe677 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -438,6 +438,10 @@ jobs: with: version: "${{ matrix.NXF_VER }}" + # For OCI containers + - name: Set XDG_RUNTIME_DIR + run: echo "XDG_RUNTIME_DIR=/run/user/$UID" >> $GITHUB_ENV + - name: Install Singularity if: matrix.profile == 'singularity' run: | From 04dd852c17660266588f3fd9ea2fd3fc19e9d9f3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 28 Nov 2023 16:02:01 -0500 Subject: [PATCH 42/84] ci: That'll do it --- .github/workflows/wave.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 51c6e8d626d..6c3ccfa4549 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -1,6 +1,7 @@ name: Build containers # When environment.yml is changed on: + push: pull_request: paths: - "**/Dockerfile" From 1127407b18f248616d3f66cb4253cc62f26bbcb0 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:09:23 -0500 Subject: [PATCH 43/84] ci: Remove Dockerfile We'll have a seperate action for this I think --- .github/workflows/wave.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 6c3ccfa4549..e689053ca91 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -4,7 +4,6 @@ on: push: pull_request: paths: - - "**/Dockerfile" - "**/environment.yml" env: From 7b241f13cfc050841ea9d4b63267e40f1fd73886 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:09:48 -0500 Subject: [PATCH 44/84] ci: Update name --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index e689053ca91..c939474e768 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -1,4 +1,4 @@ -name: Build containers +name: Wave # When environment.yml is changed on: push: From 2c3a787889da4f621796adef95b6057d63ab9490 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:24:02 -0500 Subject: [PATCH 45/84] ci: Push to the correct repos --- .github/workflows/wave.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index c939474e768..524bd476f33 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -54,13 +54,21 @@ jobs: sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave + - name: Create a registry name + uses: actions/github-script@v7 + id: registry-name + with: + result-encoding: string + script: | + return '${{ matrix.files }}'.replace('modules/nf-core', '').replace('/environment.yml', ''); + - name: Build container if: matrix.profile == 'docker' run: | wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo quay.io/nf-core/modules/bowtie \ + --build-repo ${{steps.registry-name.outputs.result}} \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -70,7 +78,7 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo quay.io/nf-core/modules/bowtie \ + --build-repo ${{steps.registry-name.outputs.result}}/singularity \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity From 895c72f5d9e50daa1a85b06304c40c141e79702f Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:30:15 -0500 Subject: [PATCH 46/84] ci: Remove OCI stuff --- .github/workflows/test.yml | 13 ++++++------- tests/config/nextflow.config | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ef8f5fe677..9a12adaf278 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -438,15 +438,11 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - # For OCI containers - - name: Set XDG_RUNTIME_DIR - run: echo "XDG_RUNTIME_DIR=/run/user/$UID" >> $GITHUB_ENV - - name: Install Singularity if: matrix.profile == 'singularity' run: | wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb - sudo apt-get install -y crun libfuse2 + sudo apt-get install -y libfuse2 sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb - name: Set up Singularity @@ -670,9 +666,12 @@ jobs: - name: Install nf-test uses: nf-core/setup-nf-test@v1 - - name: Setup apptainer + - name: Install Singularity if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main + run: | + wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb + sudo apt-get install -y libfuse2 + sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb - name: Set up Singularity if: matrix.profile == 'singularity' diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index ebe8c18284b..c24e252ec57 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,7 +14,6 @@ process { if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true - singularity.oci = true } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { From 34852d94b51ff83f338e87820dea87ddcae779bc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:35:09 -0500 Subject: [PATCH 47/84] ci: Need a full URL --- .github/workflows/wave.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 524bd476f33..bcd2b79beba 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -68,7 +68,7 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo ${{steps.registry-name.outputs.result}} \ + --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -78,7 +78,7 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo ${{steps.registry-name.outputs.result}}/singularity \ + --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}}/singularity \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity From 154fddaec6a796a3d61740e660543d3548ad6a64 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:44:18 -0500 Subject: [PATCH 48/84] ci: Fix // in container name --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index bcd2b79beba..bf2bb0b2b27 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -60,7 +60,7 @@ jobs: with: result-encoding: string script: | - return '${{ matrix.files }}'.replace('modules/nf-core', '').replace('/environment.yml', ''); + return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', ''); - name: Build container if: matrix.profile == 'docker' From f3276d10d41c9b5376c25d1800e25e99909e8fdb Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:45:15 -0500 Subject: [PATCH 49/84] ci: Remove push Build once, renovate should bump the images automagically --- .github/workflows/wave.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index bf2bb0b2b27..a9c69fde0a9 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -1,7 +1,6 @@ name: Wave # When environment.yml is changed on: - push: pull_request: paths: - "**/environment.yml" From 66ebb1443fda63b1c75354621d06068bac477213 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 10:52:24 -0500 Subject: [PATCH 50/84] build: Add containers back --- modules/nf-core/bowtie/align/main.nf | 4 +++- modules/nf-core/bowtie/build/main.nf | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 1f880aa521c..5473bf8def6 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,9 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie_align--6c5b9c93546643d8" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'nf-core/modules/bowtie/align/singularity:bowtie_align--9c6a06ebc094e52a' : + 'nf-core/modules/bowtie/align:bowtie_align--adc5d68fc5834323' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index b7e7f861c36..24e6ecf6112 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,7 +3,9 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie:bowtie_build--df26d88a69745299" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'nf-core/modules/bowtie/build/singularity:bowtie_build--7d5920d114506960' : + 'nf-core/modules/bowtie/build:bowtie_build--31571fd1e6192239' }" input: path fasta From 3394c98fd7dcaa07f58cd263728fd2936a0033e2 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 12:20:34 -0500 Subject: [PATCH 51/84] ci: Add cache repos Idk what this does exactly --- .github/workflows/wave.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index a9c69fde0a9..0efe171e5ba 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -68,6 +68,7 @@ jobs: --freeze \ --await \ --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ + --cache-repository quay.io/nf-core/wave-cache \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -78,6 +79,7 @@ jobs: --freeze \ --await \ --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}}/singularity \ + --cache-repository quay.io/nf-core/wave-cache/singularity \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity From cfc8528f209c7320d2d68f695e5027e8f0f69309 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 14:07:15 -0500 Subject: [PATCH 52/84] ci: Change registry name to use _ Because "build" is a api end point on quay.io. So `bowtie/build` doesn't work. Other plus is this matches the conda env name. --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 0efe171e5ba..ab102e1400a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -59,7 +59,7 @@ jobs: with: result-encoding: string script: | - return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', ''); + return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_'); - name: Build container if: matrix.profile == 'docker' From c20dad0c56a7655721df672fc97c56508cbf461e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 30 Nov 2023 14:17:38 -0500 Subject: [PATCH 53/84] build: / => _ in container name --- modules/nf-core/bowtie/align/main.nf | 4 ++-- modules/nf-core/bowtie/build/main.nf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 5473bf8def6..dca9c50a899 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -4,8 +4,8 @@ process BOWTIE_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'nf-core/modules/bowtie/align/singularity:bowtie_align--9c6a06ebc094e52a' : - 'nf-core/modules/bowtie/align:bowtie_align--adc5d68fc5834323' }" + 'nf-core/modules/bowtie_align/singularity:bowtie_align--42e8f624908e260f' : + 'nf-core/modules/bowtie_align:bowtie_align--dd5b3a544b28c5c3' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 24e6ecf6112..df90896708b 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -4,8 +4,8 @@ process BOWTIE_BUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'nf-core/modules/bowtie/build/singularity:bowtie_build--7d5920d114506960' : - 'nf-core/modules/bowtie/build:bowtie_build--31571fd1e6192239' }" + 'nf-core/modules/bowtie_build/singularity:bowtie_build--121b0f47850f8086' : + 'nf-core/modules/bowtie_build:bowtie_build--a6833a982c474692' }" input: path fasta From 7b0b40b1af84e982233563cee81e67ac9b1cb346 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 16 Feb 2024 09:43:47 -0600 Subject: [PATCH 54/84] Try ociAutoPull --- modules/nf-core/bowtie/align/main.nf | 4 +--- modules/nf-core/bowtie/build/main.nf | 4 +--- tests/config/nextflow.config | 3 ++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index dca9c50a899..11ac755a8a0 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,9 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'nf-core/modules/bowtie_align/singularity:bowtie_align--42e8f624908e260f' : - 'nf-core/modules/bowtie_align:bowtie_align--dd5b3a544b28c5c3' }" + container "nf-core/modules/bowtie_align:bowtie_align--dd5b3a544b28c5c3" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index df90896708b..3aa86db4a02 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,9 +3,7 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'nf-core/modules/bowtie_build/singularity:bowtie_build--121b0f47850f8086' : - 'nf-core/modules/bowtie_build:bowtie_build--a6833a982c474692' }" + container "nf-core/modules/bowtie_build:bowtie_build--a6833a982c474692" input: path fasta diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c24e252ec57..66a28f2e56d 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,6 +14,7 @@ process { if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true + singularity.ociAutoPull = true } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { @@ -41,5 +42,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.12.0-edge' } From b3488fb82a236d845ab8b9603b5da1ec630aade9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 16 Feb 2024 10:15:03 -0600 Subject: [PATCH 55/84] chore: Add renovate comments to samtools Just to trigger wave build --- modules/nf-core/samtools/view/environment.yml | 2 ++ modules/nf-core/samtools/view/main.nf | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index 150c377771b..f75f7c5e62a 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -4,5 +4,7 @@ channels: - bioconda - defaults dependencies: + # renovate: datasource=conda depName=bioconda/samtools - bioconda::samtools=1.20 + # renovate: datasource=conda depName=bioconda/htslib - bioconda::htslib=1.20 diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index 38df857604d..e45361d8593 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -3,9 +3,7 @@ process SAMTOOLS_VIEW { label 'process_low' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.20--h50ea8bc_0' : - 'biocontainers/samtools:1.20--h50ea8bc_0' }" + container "nf-core/modules/samtools_view:samtools_view--4731ee50a742fc67" input: tuple val(meta), path(input), path(index) From 5a3d54609b67b1814c545b7e3dd8b49a82cb00e6 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 16 Feb 2024 10:24:09 -0600 Subject: [PATCH 56/84] test: Add ociAutoPull to nf-test --- tests/config/nf-test.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 269f2dc3ca1..38c9236f94c 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -15,6 +15,7 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + singularity.ociAutoPull = true } conda { conda.enabled = true @@ -50,5 +51,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.12.0-edge' } From a3753e70bf29c97f70241d6d123416d373d92682 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 16 Feb 2024 11:03:01 -0600 Subject: [PATCH 57/84] ci: Bump wave version --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index ab102e1400a..7df47746a91 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -6,7 +6,7 @@ on: - "**/environment.yml" env: - WAVE_VER: "1.1.0" + WAVE_VER: "1.2.0" jobs: gen-matrix: From 0546a88160c0a538687c41903e57713ec9fb18fc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 16 Feb 2024 11:13:49 -0600 Subject: [PATCH 58/84] chore: Bump containers with new wave version Not sure why that's happening... --- modules/nf-core/bowtie/align/main.nf | 2 +- modules/nf-core/bowtie/build/main.nf | 2 +- modules/nf-core/samtools/view/main.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 11ac755a8a0..54af8c40728 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie_align:bowtie_align--dd5b3a544b28c5c3" + container "nf-core/modules/bowtie_align:bowtie_align--d797c0ccd2d330fd" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 3aa86db4a02..6d75e7625b7 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,7 +3,7 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie_build:bowtie_build--a6833a982c474692" + container "nf-core/modules/bowtie_build:bowtie_build--4ee5ca3f4bf65e6a" input: path fasta diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index e45361d8593..3a42de1cf42 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -3,7 +3,7 @@ process SAMTOOLS_VIEW { label 'process_low' conda "${moduleDir}/environment.yml" - container "nf-core/modules/samtools_view:samtools_view--4731ee50a742fc67" + container "nf-core/modules/samtools_view:samtools_view--033f764f39eb1f2d" input: tuple val(meta), path(input), path(index) From 8995941e27b4b9b934f54018aa923f38bd2f861e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 16:38:25 -0500 Subject: [PATCH 59/84] build: Update to use commity.wave.seqera.io --- modules/nf-core/bowtie/align/environment.yml | 7 +++---- modules/nf-core/bowtie/align/main.nf | 2 +- modules/nf-core/bowtie/build/environment.yml | 7 +++---- modules/nf-core/samtools/view/environment.yml | 7 +++---- modules/nf-core/samtools/view/main.nf | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/modules/nf-core/bowtie/align/environment.yml b/modules/nf-core/bowtie/align/environment.yml index fb5d7c1ef12..b48471c1022 100644 --- a/modules/nf-core/bowtie/align/environment.yml +++ b/modules/nf-core/bowtie/align/environment.yml @@ -1,8 +1,7 @@ -name: bowtie_align +name: bowtie_samtools channels: - conda-forge - bioconda - - defaults dependencies: - - bioconda::bowtie=1.3.0 - - bioconda::samtools=1.16.1 + - bioconda::bowtie=1.3.1 + - bioconda::samtools=1.20 diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 54af8c40728..35b80d2c26d 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,7 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie_align:bowtie_align--d797c0ccd2d330fd" + container "community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/environment.yml b/modules/nf-core/bowtie/build/environment.yml index 3b6490b87e5..b48471c1022 100644 --- a/modules/nf-core/bowtie/build/environment.yml +++ b/modules/nf-core/bowtie/build/environment.yml @@ -1,8 +1,7 @@ -name: bowtie_build +name: bowtie_samtools channels: - conda-forge - bioconda - - defaults dependencies: - - bioconda::bowtie=1.3.0 - - bioconda::samtools=1.16.1 + - bioconda::bowtie=1.3.1 + - bioconda::samtools=1.20 diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index f75f7c5e62a..1c255c63d7f 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -1,10 +1,9 @@ -name: samtools_view +name: htslib_samtools channels: - conda-forge - bioconda - - defaults dependencies: - # renovate: datasource=conda depName=bioconda/samtools - - bioconda::samtools=1.20 # renovate: datasource=conda depName=bioconda/htslib - bioconda::htslib=1.20 + # renovate: datasource=conda depName=bioconda/samtools + - bioconda::samtools=1.20 diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index 3a42de1cf42..3f273468d5c 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -3,7 +3,7 @@ process SAMTOOLS_VIEW { label 'process_low' conda "${moduleDir}/environment.yml" - container "nf-core/modules/samtools_view:samtools_view--033f764f39eb1f2d" + container "community.wave.seqera.io/library/htslib_samtools:1.20--11a4e6daa46930ec" input: tuple val(meta), path(input), path(index) From 08292ea118f4c646df61d06435b86eff7c099339 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 16:48:17 -0500 Subject: [PATCH 60/84] ci: Bump wave-cli to 1.4.1 --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 7df47746a91..1126adbf079 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -6,7 +6,7 @@ on: - "**/environment.yml" env: - WAVE_VER: "1.2.0" + WAVE_VER: "1.4.1" jobs: gen-matrix: From bc4a8bcc626c5e825b0ee74ca5984e7d10c2b70e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 17:05:08 -0500 Subject: [PATCH 61/84] ci: Try apptainer --- .github/workflows/test.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a12adaf278..e3977dfec5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -438,12 +438,9 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Install Singularity + - name: Setup apptainer if: matrix.profile == 'singularity' - run: | - wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb - sudo apt-get install -y libfuse2 - sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb + uses: eWaterCycle/setup-apptainer@main - name: Set up Singularity if: matrix.profile == 'singularity' @@ -666,12 +663,9 @@ jobs: - name: Install nf-test uses: nf-core/setup-nf-test@v1 - - name: Install Singularity + - name: Setup apptainer if: matrix.profile == 'singularity' - run: | - wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb - sudo apt-get install -y libfuse2 - sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb + uses: eWaterCycle/setup-apptainer@main - name: Set up Singularity if: matrix.profile == 'singularity' From 300502d81a67b540a720cc342f21fbe563117dbd Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 17:31:37 -0500 Subject: [PATCH 62/84] ci: Remove build-repo to see what happens --- .github/workflows/wave.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 1126adbf079..fde4414ccb3 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -67,7 +67,6 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ --cache-repository quay.io/nf-core/wave-cache \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -78,7 +77,6 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}}/singularity \ --cache-repository quay.io/nf-core/wave-cache/singularity \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ From 69e1ea59a079752946682a04c2a3c066b21bc283 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 17:37:09 -0500 Subject: [PATCH 63/84] build: Bump Nextflow version requirement --- tests/config/nextflow.config | 2 +- tests/config/nf-test.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 66a28f2e56d..b483bd914fe 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -42,5 +42,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.12.0-edge' + nextflowVersion = '!>=24.04.2' } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 38c9236f94c..edde3b5dd4e 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -51,5 +51,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.12.0-edge' + nextflowVersion = '!>=24.04.2' } From 14aa2958b55d1fd513c5cf87dab98a5fedcbe15d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 17:52:10 -0500 Subject: [PATCH 64/84] fix: Get rid of the environment name? Maybe this will get the auto generated tag? --- modules/nf-core/samtools/view/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index 1c255c63d7f..114ba2e2eed 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -1,4 +1,3 @@ -name: htslib_samtools channels: - conda-forge - bioconda From ca5d48a0a6d94fffea6a499dc576ad0663dfdb56 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 17:55:03 -0500 Subject: [PATCH 65/84] ci: Bump action versions --- .github/workflows/wave.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index fde4414ccb3..1ef5f6750cb 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -14,12 +14,11 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - name: Calculate file differences id: diff - uses: tj-actions/changed-files@v35 + uses: tj-actions/changed-files@v44 with: json: true quotepath: false @@ -44,8 +43,7 @@ jobs: fail-fast: false matrix: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" steps: - # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - name: Install wave-cli run: | From 4353d08d8469b19d19f2fe8ec991622f1b5a8761 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 18:07:30 -0500 Subject: [PATCH 66/84] ci: Try name-strategy tagPrefix https://github.com/seqeralabs/wave-cli/commit/269df0e52e18c9dbd919a5a5b8934a37b5300ad1 --- .github/workflows/wave.yml | 1 + modules/nf-core/samtools/view/environment.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 1ef5f6750cb..67827513797 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -63,6 +63,7 @@ jobs: if: matrix.profile == 'docker' run: | wave --conda-file "${{ matrix.files }}" \ + --name-strategy tagPrefix \ --freeze \ --await \ --cache-repository quay.io/nf-core/wave-cache \ diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index 114ba2e2eed..1c255c63d7f 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -1,3 +1,4 @@ +name: htslib_samtools channels: - conda-forge - bioconda From b36b63193365bcaa9387f5e1e7e2028f815a8f37 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 18:10:53 -0500 Subject: [PATCH 67/84] ci: Remove singularity build for now --- .github/workflows/wave.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 67827513797..9be58feff5a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -70,15 +70,15 @@ jobs: --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} - - name: Build Singularity - if: matrix.profile == 'singularity' - run: | - wave --conda-file "${{ matrix.files }}" \ - --freeze \ - --await \ - --cache-repository quay.io/nf-core/wave-cache/singularity \ - --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ - --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ - --singularity + # - name: Build Singularity + # if: matrix.profile == 'singularity' + # run: | + # wave --conda-file "${{ matrix.files }}" \ + # --freeze \ + # --await \ + # --cache-repository quay.io/nf-core/wave-cache/singularity \ + # --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ + # --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ + # --singularity # TODO Build from Dockerfiles From d0d831505d59b9b2df0a5e6ece4af41491620252 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 18:12:56 -0500 Subject: [PATCH 68/84] ci: Try imageSuffix --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 9be58feff5a..ffee2f9613f 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -63,7 +63,7 @@ jobs: if: matrix.profile == 'docker' run: | wave --conda-file "${{ matrix.files }}" \ - --name-strategy tagPrefix \ + --name-strategy imageSuffix \ --freeze \ --await \ --cache-repository quay.io/nf-core/wave-cache \ From 2112a96c1f4cff0a956372c07b3408e9cadaad73 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 18:17:18 -0500 Subject: [PATCH 69/84] ci: Try none --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index ffee2f9613f..cfdf6faf93a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -63,7 +63,7 @@ jobs: if: matrix.profile == 'docker' run: | wave --conda-file "${{ matrix.files }}" \ - --name-strategy imageSuffix \ + --name-strategy none \ --freeze \ --await \ --cache-repository quay.io/nf-core/wave-cache \ From 5ef863f0788ad87db3b0f29092de02e157ce5b02 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 20:55:05 -0500 Subject: [PATCH 70/84] ci: What is the bowtie container name --- modules/nf-core/bowtie/align/environment.yml | 2 ++ modules/nf-core/bowtie/build/environment.yml | 2 ++ modules/nf-core/bowtie/build/main.nf | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/bowtie/align/environment.yml b/modules/nf-core/bowtie/align/environment.yml index b48471c1022..78888838a41 100644 --- a/modules/nf-core/bowtie/align/environment.yml +++ b/modules/nf-core/bowtie/align/environment.yml @@ -3,5 +3,7 @@ channels: - conda-forge - bioconda dependencies: + # renovate: datasource=conda depName=bioconda/bowtie - bioconda::bowtie=1.3.1 + # renovate: datasource=conda depName=bioconda/samtools - bioconda::samtools=1.20 diff --git a/modules/nf-core/bowtie/build/environment.yml b/modules/nf-core/bowtie/build/environment.yml index b48471c1022..78888838a41 100644 --- a/modules/nf-core/bowtie/build/environment.yml +++ b/modules/nf-core/bowtie/build/environment.yml @@ -3,5 +3,7 @@ channels: - conda-forge - bioconda dependencies: + # renovate: datasource=conda depName=bioconda/bowtie - bioconda::bowtie=1.3.1 + # renovate: datasource=conda depName=bioconda/samtools - bioconda::samtools=1.20 diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 6d75e7625b7..2107cf06371 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,7 +3,7 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "nf-core/modules/bowtie_build:bowtie_build--4ee5ca3f4bf65e6a" + container "community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4" input: path fasta From ec74401a94b27953c1e07110b3ac68b864898d16 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 11 Jun 2024 21:42:57 -0500 Subject: [PATCH 71/84] ci: Remove --name-strategy --- .github/workflows/wave.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index cfdf6faf93a..847a49a4560 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -63,7 +63,6 @@ jobs: if: matrix.profile == 'docker' run: | wave --conda-file "${{ matrix.files }}" \ - --name-strategy none \ --freeze \ --await \ --cache-repository quay.io/nf-core/wave-cache \ From 364f7a669fa188ae66088f45272fd0a97dab765c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 18 Jun 2024 13:52:01 -0500 Subject: [PATCH 72/84] style: Add back in container elvis operator --- .github/workflows/wave.yml | 20 +++++++++---------- modules/nf-core/bowtie/align/environment.yml | 1 - modules/nf-core/bowtie/align/main.nf | 5 ++++- modules/nf-core/bowtie/build/environment.yml | 1 - modules/nf-core/bowtie/build/main.nf | 4 +++- modules/nf-core/samtools/view/environment.yml | 1 - modules/nf-core/samtools/view/main.nf | 4 +++- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 847a49a4560..1ef5f6750cb 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -69,15 +69,15 @@ jobs: --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} - # - name: Build Singularity - # if: matrix.profile == 'singularity' - # run: | - # wave --conda-file "${{ matrix.files }}" \ - # --freeze \ - # --await \ - # --cache-repository quay.io/nf-core/wave-cache/singularity \ - # --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ - # --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ - # --singularity + - name: Build Singularity + if: matrix.profile == 'singularity' + run: | + wave --conda-file "${{ matrix.files }}" \ + --freeze \ + --await \ + --cache-repository quay.io/nf-core/wave-cache/singularity \ + --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ + --singularity # TODO Build from Dockerfiles diff --git a/modules/nf-core/bowtie/align/environment.yml b/modules/nf-core/bowtie/align/environment.yml index 78888838a41..61bd69c2c7a 100644 --- a/modules/nf-core/bowtie/align/environment.yml +++ b/modules/nf-core/bowtie/align/environment.yml @@ -1,4 +1,3 @@ -name: bowtie_samtools channels: - conda-forge - bioconda diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 35b80d2c26d..4d5b94e625f 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,10 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4" + container "" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'oras://community.wave.seqera.io/library/bowtie_samtools:16f00b34cfc72399' : + 'community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/environment.yml b/modules/nf-core/bowtie/build/environment.yml index 78888838a41..61bd69c2c7a 100644 --- a/modules/nf-core/bowtie/build/environment.yml +++ b/modules/nf-core/bowtie/build/environment.yml @@ -1,4 +1,3 @@ -name: bowtie_samtools channels: - conda-forge - bioconda diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 2107cf06371..1c2f7f04fb1 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -3,7 +3,9 @@ process BOWTIE_BUILD { label 'process_high' conda "${moduleDir}/environment.yml" - container "community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'oras://community.wave.seqera.io/library/bowtie_samtools:16f00b34cfc72399' : + 'community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4' }" input: path fasta diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index 1c255c63d7f..114ba2e2eed 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -1,4 +1,3 @@ -name: htslib_samtools channels: - conda-forge - bioconda diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index 3f273468d5c..03ce060c74a 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -3,7 +3,9 @@ process SAMTOOLS_VIEW { label 'process_low' conda "${moduleDir}/environment.yml" - container "community.wave.seqera.io/library/htslib_samtools:1.20--11a4e6daa46930ec" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'oras://community.wave.seqera.io/library/htslib_samtools:1.20--9fb9031594b6902c' : + 'community.wave.seqera.io/library/htslib_samtools:1.20--11a4e6daa46930ec' }" input: tuple val(meta), path(input), path(index) From a1e54b4f091affbdeb9f5d525296951eb4aafa94 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 18 Jun 2024 14:03:33 -0500 Subject: [PATCH 73/84] ci: Remove cache repo --- .github/workflows/wave.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 1ef5f6750cb..09125cda59a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -65,7 +65,6 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --cache-repository quay.io/nf-core/wave-cache \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} @@ -75,7 +74,6 @@ jobs: wave --conda-file "${{ matrix.files }}" \ --freeze \ --await \ - --cache-repository quay.io/nf-core/wave-cache/singularity \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity From 8fa7b0d2d2cdfbd70575a63049e1b72f40c288cd Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 18 Jun 2024 14:17:09 -0500 Subject: [PATCH 74/84] Revert "build: Bump Nextflow version requirement" This reverts commit 69e1ea59a079752946682a04c2a3c066b21bc283. --- tests/config/nextflow.config | 2 +- tests/config/nf-test.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index b483bd914fe..66a28f2e56d 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -42,5 +42,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=23.12.0-edge' } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index edde3b5dd4e..38c9236f94c 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -51,5 +51,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=23.12.0-edge' } From 727dda0cea81652f63a1431ef54e50c67e10fe5c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 18 Jun 2024 14:17:22 -0500 Subject: [PATCH 75/84] Revert "test: Add ociAutoPull to nf-test" This reverts commit 5a3d54609b67b1814c545b7e3dd8b49a82cb00e6. --- tests/config/nextflow.config | 3 +-- tests/config/nf-test.config | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 66a28f2e56d..c24e252ec57 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,7 +14,6 @@ process { if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true - singularity.ociAutoPull = true } else if ("$PROFILE" == "conda") { conda.enabled = true } else if ("$PROFILE" == "mamba") { @@ -42,5 +41,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.12.0-edge' + nextflowVersion = '!>=23.04.0' } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 38c9236f94c..269f2dc3ca1 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -15,7 +15,6 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true - singularity.ociAutoPull = true } conda { conda.enabled = true @@ -51,5 +50,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.12.0-edge' + nextflowVersion = '!>=23.04.0' } From daa44fe4e0b5c32843beae3e65eaffb58c281193 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 10 Sep 2024 11:49:33 -0500 Subject: [PATCH 76/84] test(#6505): Snapshot the versions contents, not the hash --- .../nf-core/bowtie/build/tests/main.nf.test | 59 ++++++++++ .../bowtie/build/tests/main.nf.test.snap | 110 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 modules/nf-core/bowtie/build/tests/main.nf.test create mode 100644 modules/nf-core/bowtie/build/tests/main.nf.test.snap diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test b/modules/nf-core/bowtie/build/tests/main.nf.test new file mode 100644 index 00000000000..81209a7b1fc --- /dev/null +++ b/modules/nf-core/bowtie/build/tests/main.nf.test @@ -0,0 +1,59 @@ +nextflow_process { + + name "Test Process BOWTIE_BUILD" + script "../main.nf" + process "BOWTIE_BUILD" + + tag "modules" + tag "modules_nfcore" + tag "bowtie" + tag "bowtie/build" + + test("sarscov2 - fasta") { + + when { + process { + """ + input[0] = [ + [id: 'sarscov2'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("sarscov2 - fasta - stub") { + + options "-stub" + tag "stub" + + when { + process { + """ + input[0] = [[id: 'sarscov2'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot(path(process.out.versions.get(0)).yaml).match("versions") }, + ) + } + + } + +} diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test.snap b/modules/nf-core/bowtie/build/tests/main.nf.test.snap new file mode 100644 index 00000000000..6951ac5b5d4 --- /dev/null +++ b/modules/nf-core/bowtie/build/tests/main.nf.test.snap @@ -0,0 +1,110 @@ +{ + "sarscov2 - fasta - stub": { + "content": [ + { + "0": [ + [ + { + "id": "sarscov2" + }, + [ + "sarscov2.1.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.2.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.3.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.4.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.rev.1.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.rev.2.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + ], + "index": [ + [ + { + "id": "sarscov2" + }, + [ + "sarscov2.1.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.2.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.3.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.4.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.rev.1.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sarscov2.rev.2.ebwt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-18T08:38:14.852528155" + }, + "versions": { + "content": [ + { + "BOWTIE_BUILD": { + "bowtie": "1.3.0" + } + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-09-10T11:48:00.007395903" + }, + "sarscov2 - fasta": { + "content": [ + { + "0": [ + [ + { + "id": "sarscov2" + }, + [ + "sarscov2.1.ebwt:md5,d9b76ecf9fd0413240173273b38d8199", + "sarscov2.2.ebwt:md5,02b44af9f94c62ecd3c583048e25d4cf", + "sarscov2.3.ebwt:md5,4ed93abba181d8dfab2e303e33114777", + "sarscov2.4.ebwt:md5,c25be5f8b0378abf7a58c8a880b87626", + "sarscov2.rev.1.ebwt:md5,b37aaf11853e65a3b13561f27a912b06", + "sarscov2.rev.2.ebwt:md5,9e6b0c4c1ddb99ae71ff8a4fe5ec6459" + ] + ] + ], + "1": [ + "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + ], + "index": [ + [ + { + "id": "sarscov2" + }, + [ + "sarscov2.1.ebwt:md5,d9b76ecf9fd0413240173273b38d8199", + "sarscov2.2.ebwt:md5,02b44af9f94c62ecd3c583048e25d4cf", + "sarscov2.3.ebwt:md5,4ed93abba181d8dfab2e303e33114777", + "sarscov2.4.ebwt:md5,c25be5f8b0378abf7a58c8a880b87626", + "sarscov2.rev.1.ebwt:md5,b37aaf11853e65a3b13561f27a912b06", + "sarscov2.rev.2.ebwt:md5,9e6b0c4c1ddb99ae71ff8a4fe5ec6459" + ] + ] + ], + "versions": [ + "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-18T08:37:53.65689025" + } +} \ No newline at end of file From 457bf9e9a72984baa82bd28f04e49637b721975b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 10 Sep 2024 11:55:22 -0500 Subject: [PATCH 77/84] ci(#6505): Update version snapshot after building containers --- .github/workflows/wave.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 09125cda59a..c6484852899 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -78,4 +78,41 @@ jobs: --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity + - name: Bump Versions + env: + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + run: | + # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker + if [ "${{ matrix.profile }}" == "docker" ]; then + PROFILE="docker_self_hosted" + else + PROFILE=${{ matrix.profile }} + fi + + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ matrix.profile }} \ + --tap=test.tap \ + # --ci \ + --verbose \ + --only-changed \ + --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ + --filter ${{ matrix.filter }} \ + --follow-dependencies \ + --tag version \ + --update-snapshot + + - name: Commit & push version bumps + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git config push.default upstream + git add . + git status + git commit -m "[automated] Bump versions snapshot" + git push + # TODO Build from Dockerfiles From 2925c173deee2a797d21f0d965255a25e7a5274e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 10 Sep 2024 12:06:38 -0500 Subject: [PATCH 78/84] test(#6505): Attempt a topic channel with tests https://github.com/askimed/nf-test/issues/258 --- modules/nf-core/bowtie/build/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 1c2f7f04fb1..68508fd9353 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -13,6 +13,7 @@ process BOWTIE_BUILD { output: path 'bowtie' , emit: index path "versions.yml" , emit: versions + tuple val("$task.process"), val("bowtie"), eval("echo \$(bowtie --version 2>&1) | sed 's/^.*bowtie-align-s version //; s/ .*\$//'"), topic: version when: task.ext.when == null || task.ext.when From 2462a112ee42f16cf555365c244d38a703cb3999 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 30 Oct 2024 12:07:30 +0100 Subject: [PATCH 79/84] chore: Bump to 1.5.0 --- .github/workflows/wave.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index c6484852899..1cd83260a09 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -6,7 +6,7 @@ on: - "**/environment.yml" env: - WAVE_VER: "1.4.1" + WAVE_VER: "1.5.0" jobs: gen-matrix: From 6934f0ba033bcfe128dc2334c1b2a4318f86bc0e Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 30 Oct 2024 12:07:48 +0100 Subject: [PATCH 80/84] fix: Remove shard and filter on test bumping --- .github/workflows/wave.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 1cd83260a09..26af8929031 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -99,8 +99,8 @@ jobs: # --ci \ --verbose \ --only-changed \ - --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ - --filter ${{ matrix.filter }} \ + # --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ + # --filter ${{ matrix.filter }} \ --follow-dependencies \ --tag version \ --update-snapshot From 3fa2c06ca4708f9fae67924703b4530b6cde440b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 30 Oct 2024 12:49:23 +0100 Subject: [PATCH 81/84] build: Bump images to match environment --- modules/nf-core/bowtie/align/main.nf | 4 ++-- modules/nf-core/samtools/view/main.nf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index e9532305763..ab0a10a3591 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -5,8 +5,8 @@ process BOWTIE_ALIGN { conda "${moduleDir}/environment.yml" container "" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'oras://community.wave.seqera.io/library/bowtie_samtools:16f00b34cfc72399' : - 'community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : + 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index 37e05cec887..41fa3d6a0b9 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_VIEW { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.21--h50ea8bc_0' : - 'biocontainers/samtools:1.21--h50ea8bc_0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9e/9edc2564215d5cd137a8b25ca8a311600987186d406b092022444adf3c4447f7/data' : + 'community​.wave​.seqera​.io/library/htslib_samtools:1​.21--6cb89bfd40cbaabf' }" input: tuple val(meta), path(input), path(index) From 8c73e7133535f40a286b4484a0336f84daae9ce7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 30 Oct 2024 13:00:19 +0100 Subject: [PATCH 82/84] ci: Fix nf-test setup --- .github/workflows/wave.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 26af8929031..7e860230828 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -5,8 +5,12 @@ on: paths: - "**/environment.yml" +# TODO On complete call testing CI +# TODO Skip testing CI if any changes to environment.yml + env: WAVE_VER: "1.5.0" + NFTEST_VER: "0.9.1" jobs: gen-matrix: @@ -78,10 +82,21 @@ jobs: --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity - - name: Bump Versions + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + + - uses: nf-core/setup-nextflow@v2 + + - uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFTEST_VER }} + + - name: Bump Snapshot Versions env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + # NFT_DIFF: "pdiff" + # NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" run: | From 24de614adb5ec6d33baf6a35175e24e8273f921a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 30 Oct 2024 15:33:55 +0100 Subject: [PATCH 83/84] ci: Remove snapshot bumping --- .github/workflows/wave.yml | 101 +++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 48 deletions(-) diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 7e860230828..d450ef24d34 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -82,52 +82,57 @@ jobs: --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ --singularity - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - uses: nf-core/setup-nextflow@v2 - - - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Bump Snapshot Versions - env: - # NFT_DIFF: "pdiff" - # NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }} \ - --tap=test.tap \ - # --ci \ - --verbose \ - --only-changed \ - # --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ - # --filter ${{ matrix.filter }} \ - --follow-dependencies \ - --tag version \ - --update-snapshot - - - name: Commit & push version bumps - run: | - git config user.email "core@nf-co.re" - git config user.name "nf-core-bot" - git config push.default upstream - git add . - git status - git commit -m "[automated] Bump versions snapshot" - git push - # TODO Build from Dockerfiles + + # bump-versions: + # needs: gen-matrix + # name: bump-versions + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + # with: + # distribution: "temurin" + # java-version: "17" + + # - uses: nf-core/setup-nextflow@v2 + + # - uses: nf-core/setup-nf-test@v1 + # with: + # version: ${{ env.NFTEST_VER }} + + # - name: Bump Snapshot Versions + # env: + # # NFT_DIFF: "pdiff" + # # NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + # SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + # SENTIEON_AUTH_MECH: "GitHub Actions - token" + # run: | + # # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker + # if [ "${{ matrix.profile }}" == "docker" ]; then + # PROFILE="docker_self_hosted" + # else + # PROFILE=${{ matrix.profile }} + # fi + + # NFT_WORKDIR=~ \ + # nf-test test \ + # --profile=${{ matrix.profile }} \ + # --tap=test.tap \ + # # --ci \ + # --verbose \ + # --only-changed \ + # # --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ + # # --filter ${{ matrix.filter }} \ + # --follow-dependencies \ + # --tag version \ + # --update-snapshot + + # - name: Commit & push version bumps + # run: | + # git config user.email "core@nf-co.re" + # git config user.name "nf-core-bot" + # git config push.default upstream + # git add . + # git status + # git commit -m "[automated] Bump versions snapshot" + # git push From 86ad692f43609a5aa82e76e392a82cf4071969da Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 7 Nov 2024 09:56:47 -0600 Subject: [PATCH 84/84] build: Fix containers in bowtie --- modules/nf-core/bowtie/align/main.nf | 1 - modules/nf-core/bowtie/build/main.nf | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index ab0a10a3591..3dee85ffb4d 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -3,7 +3,6 @@ process BOWTIE_ALIGN { label 'process_high' conda "${moduleDir}/environment.yml" - container "" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 23e9fb23dd6..0f6b9d4d1bf 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -4,8 +4,8 @@ process BOWTIE_BUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'oras://community.wave.seqera.io/library/bowtie_samtools:16f00b34cfc72399' : - 'community.wave.seqera.io/library/bowtie_samtools:772b3bee982574e4' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : + 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(fasta)