diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 0000000..38d2d7f
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,120 @@
+name: Build
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+on:
+ push:
+ branches: [ main ]
+ workflow_dispatch:
+ inputs:
+ version:
+ description: |
+ The version of the project to build. Example: `1.0.3`.
+
+ If not provided, a development build with a version name
+ based on the branch name will be built. Otherwise, a release
+ build with the provided version will be built.
+ required: false
+
+jobs:
+ # phase 1
+ target:
+ name: Build target branch
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+
+ outputs:
+ target_branch: ${{ steps.build-target.outputs.target_branch }}
+ version: ${{ steps.build-target.outputs.version }}
+ docker_matrix: ${{ steps.build-target.outputs.docker_matrix }}
+
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+ with:
+ submodules: 'recursive'
+ fetch-depth: 0
+
+ - name: Install Viash
+ uses: viash-io/viash-actions/setup@v6
+
+ - name: Determine variables
+ id: variables
+ run: |
+ VERSION="${{ github.event.inputs.version }}"
+ SOURCE_BRANCH=$(echo "$GITHUB_REF" | sed 's/refs\/heads\///')
+
+ if [[ -z $VERSION ]]; then
+ TARGET_BRANCH="build/$SOURCE_BRANCH"
+ VERSION=$(echo "$TARGET_BRANCH" | sed 's/[^a-zA-Z0-9_]/_/g')
+ else
+ if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
+ echo "Version '$VERSION' does not match PEP440"
+ exit 1
+ fi
+ TARGET_BRANCH="release/${VERSION%.*}.x"
+ fi
+
+ echo "Set version of Viash package to '$VERSION'"
+ echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+ echo "Set target branch to '$TARGET_BRANCH'"
+ echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT
+
+
+ - uses: viash-io/viash-actions/project/build-target@v6
+ id: build-target
+ with:
+ target_branch: ${{ steps.variables.outputs.target_branch }}
+ version: ${{ steps.variables.outputs.version }}
+
+ - name: Deploy to target branch
+ uses: peaceiris/actions-gh-pages@v4
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ publish_branch: ${{ steps.variables.outputs.target_branch }}
+ publish_dir: .
+
+ # phase 2
+ docker:
+ name: Build and push Docker images
+ needs: target
+
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+
+ strategy:
+ fail-fast: false
+ matrix:
+ component: ${{ fromJson(needs.target.outputs.docker_matrix) }}
+
+ steps:
+ # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+ - uses: data-intuitive/reclaim-the-bytes@v2
+
+ - uses: actions/checkout@v4
+ with:
+ submodules: 'recursive'
+ ref: ${{ needs.target.outputs.target_branch }}
+
+ - uses: viash-io/viash-actions/setup@v6
+
+ - name: Login to container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Build image
+ run: |
+ ${{matrix.component.executable}} ---engine docker ---setup build ---verbose
+
+ - name: Push image
+ run: |
+ ${{matrix.component.executable}} ---engine docker ---setup push ---verbose
diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml
deleted file mode 100644
index 2772dc6..0000000
--- a/.github/workflows/main-build.yml
+++ /dev/null
@@ -1,93 +0,0 @@
-name: main build
-concurrency: main_build
-
-on:
- push:
- branches: [ 'main' ]
-
-jobs:
- # phase 1
- list:
- runs-on: ubuntu-latest
-
- outputs:
- component_matrix: ${{ steps.set_matrix.outputs.matrix }}
- cache_key: ${{ steps.cache.outputs.cache_key }}
-
- steps:
- - uses: actions/checkout@v3
-
- - uses: viash-io/viash-actions/setup@v3
-
- - name: Remove target folder from .gitignore
- run: |
- # allow publishing the target folder
- sed -i '/^target.*/d' .gitignore
-
- - uses: viash-io/viash-actions/ns-build@v3
- with:
- config_mod: .functionality.version := 'main_build'
- parallel: true
-
- - name: Deploy to target branch
- uses: peaceiris/actions-gh-pages@v3
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
- publish_dir: .
- publish_branch: main_build
-
- - id: ns_list
- uses: viash-io/viash-actions/ns-list@v3
- with:
- platform: docker
- src: src
- format: json
-
- - id: set_matrix
- run: |
- echo "matrix=$(jq -c '[ .[] |
- {
- "name": (.functionality.namespace + "/" + .functionality.name),
- "config": .info.config,
- "dir": .info.config | capture("^(?
.*\/)").dir
- }
- ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
- # phase 2
- build:
- needs: list
-
- runs-on: ubuntu-latest
-
- strategy:
- fail-fast: false
- matrix:
- component: ${{ fromJson(needs.list.outputs.component_matrix) }}
-
- steps:
- - uses: actions/checkout@v3
-
- - uses: viash-io/viash-actions/setup@v3
-
- - name: Build container
- uses: viash-io/viash-actions/ns-build@v3
- with:
- config_mod: .functionality.version := 'main_build'
- platform: docker
- src: ${{ matrix.component.dir }}
- setup: build
-
- - name: Login to container registry
- uses: docker/login-action@v2
- with:
- registry: ghcr.io
- username: ${{ secrets.GTHB_USER }}
- password: ${{ secrets.GTHB_PAT }}
-
- - name: Push container
- uses: viash-io/viash-actions/ns-build@v3
- with:
- config_mod: .functionality.version := 'main_build'
- platform: docker
- src: ${{ matrix.component.dir }}
- setup: push
\ No newline at end of file
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..91aa083
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,90 @@
+name: Test
+
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+
+jobs:
+ # phase 1
+ list:
+ name: List components to test
+ runs-on: ubuntu-latest
+
+ outputs:
+ matrix: ${{ steps.ns_list_filter_nextflow.outputs.output_matrix }}
+ cache_key: ${{ steps.cache.outputs.cache_key }}
+ dest_paths: ${{ steps.cache.outputs.dest_paths }}
+
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ submodules: 'recursive'
+
+ - uses: viash-io/viash-actions/setup@v6
+
+ - uses: viash-io/viash-actions/project/sync-and-cache@v6
+ id: cache
+
+ - id: ns_list
+ uses: viash-io/viash-actions/ns-list@v6
+ with:
+ format: json
+ runner: executable
+
+ - id: ns_list_filter_changed
+ uses: viash-io/viash-actions/project/detect-changed-components@v6
+ with:
+ input_file: "${{ steps.ns_list.outputs.output_file }}"
+
+ - name: Filter out Nextflow scripts (testing currently not supported)
+ id: ns_list_filter_nextflow
+ run: |
+ OUTPUT_MATRIX=$(echo '${{ steps.ns_list_filter_changed.outputs.output_matrix }}' | jq -c 'map(select(.main_script_type != "nextflow_script"))')
+ echo "output_matrix=$OUTPUT_MATRIX" >> $GITHUB_OUTPUT
+
+ # phase 2
+ test:
+ name: Test component
+ needs: list
+ if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
+ runs-on: ubuntu-latest
+
+ strategy:
+ fail-fast: false
+ matrix:
+ component: ${{ fromJson(needs.list.outputs.matrix) }}
+
+ steps:
+ # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+ - name: Clear space
+ uses: data-intuitive/reclaim-the-bytes@v2
+
+ - name: Check out repository
+ uses: actions/checkout@v4
+ with:
+ submodules: 'recursive'
+ fetch-depth: 0
+
+ - name: Install Viash
+ uses: viash-io/viash-actions/setup@v6
+
+ # use cache
+ - name: Cache resources data
+ if: ${{ needs.list.outputs.cache_key != '' }}
+ uses: actions/cache/restore@v4
+ timeout-minutes: 10
+ with:
+ path: ${{ needs.list.outputs.dest_paths }}
+ key: ${{ needs.list.outputs.cache_key }}
+
+ - name: Run test
+ timeout-minutes: 30
+ env:
+ VIASH_TEMP: "${{ runner.temp }}/viash_test_temp"
+ run: |
+ viash test \
+ "${{ matrix.component.config }}" \
+ --cpus 2 \
+ --memory "16gb"
diff --git a/.github/workflows/viash-test.yml b/.github/workflows/viash-test.yml
deleted file mode 100644
index 9a431c2..0000000
--- a/.github/workflows/viash-test.yml
+++ /dev/null
@@ -1,97 +0,0 @@
-name: viash test
-
-on:
- pull_request:
- push:
- branches: [ '**' ]
-
-jobs:
- run_ci_check_job:
- runs-on: ubuntu-latest
- outputs:
- run_ci: ${{ steps.github_cli.outputs.check }}
- steps:
- - name: 'Check if branch has an existing pull request and the trigger was a push'
- id: github_cli
- run: |
- pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
- # If the branch has a PR and this run was triggered by a push event, do not run
- if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
- echo "check=false" >> $GITHUB_OUTPUT
- else
- echo "check=true" >> $GITHUB_OUTPUT
- fi
- env:
- GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}
-
- # phase 1
- list:
- needs: run_ci_check_job
- env:
- s3_bucket: s3://openpipelines-data/
- runs-on: ubuntu-latest
- if: "needs.run_ci_check_job.outputs.run_ci == 'true'"
-
- outputs:
- matrix: ${{ steps.set_matrix.outputs.matrix }}
- cache_key: ${{ steps.cache.outputs.cache_key }}
-
- steps:
- - uses: actions/checkout@v3
- with:
- fetch-depth: 0
-
- - uses: viash-io/viash-actions/setup@v3
-
- - name: Get changed files
- id: changed-files
- uses: tj-actions/changed-files@v35.6.4
- with:
- separator: ";"
- diff_relative: true
-
- - id: ns_list
- uses: viash-io/viash-actions/ns-list@v3
- with:
- platform: docker
- format: json
-
- - id: ns_list_filtered
- uses: viash-io/viash-actions/project/detect-changed-components@v3
- with:
- input_file: "${{ steps.ns_list.outputs.output_file }}"
-
- - id: set_matrix
- run: |
- echo "matrix=$(jq -c '[ .[] |
- {
- "name": (.functionality.namespace + "/" + .functionality.name),
- "config": .info.config,
- "dir": .info.config | capture("^(?.*\/)").dir
- }
- ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
- # phase 2
- viash_test:
- needs: list
- if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
- runs-on: ubuntu-latest
-
- strategy:
- fail-fast: false
- matrix:
- component: ${{ fromJson(needs.list.outputs.matrix) }}
-
- steps:
- - uses: actions/checkout@v3
-
- - uses: viash-io/viash-actions/setup@v3
-
- - name: Run test
- timeout-minutes: 30
- run: |
- viash test \
- "${{ matrix.component.config }}" \
- --cpus 2 \
- --memory "5gb"
-