diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..38d2d7f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,120 @@ +name: Build + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [ main ] + workflow_dispatch: + inputs: + version: + description: | + The version of the project to build. Example: `1.0.3`. + + If not provided, a development build with a version name + based on the branch name will be built. Otherwise, a release + build with the provided version will be built. + required: false + +jobs: + # phase 1 + target: + name: Build target branch + runs-on: ubuntu-latest + permissions: + contents: write + + outputs: + target_branch: ${{ steps.build-target.outputs.target_branch }} + version: ${{ steps.build-target.outputs.version }} + docker_matrix: ${{ steps.build-target.outputs.docker_matrix }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 + + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Determine variables + id: variables + run: | + VERSION="${{ github.event.inputs.version }}" + SOURCE_BRANCH=$(echo "$GITHUB_REF" | sed 's/refs\/heads\///') + + if [[ -z $VERSION ]]; then + TARGET_BRANCH="build/$SOURCE_BRANCH" + VERSION=$(echo "$TARGET_BRANCH" | sed 's/[^a-zA-Z0-9_]/_/g') + else + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then + echo "Version '$VERSION' does not match PEP440" + exit 1 + fi + TARGET_BRANCH="release/${VERSION%.*}.x" + fi + + echo "Set version of Viash package to '$VERSION'" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + echo "Set target branch to '$TARGET_BRANCH'" + echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT + + + - uses: viash-io/viash-actions/project/build-target@v6 + id: build-target + with: + target_branch: ${{ steps.variables.outputs.target_branch }} + version: ${{ steps.variables.outputs.version }} + + - name: Deploy to target branch + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: ${{ steps.variables.outputs.target_branch }} + publish_dir: . + + # phase 2 + docker: + name: Build and push Docker images + needs: target + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + component: ${{ fromJson(needs.target.outputs.docker_matrix) }} + + steps: + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - uses: data-intuitive/reclaim-the-bytes@v2 + + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + ref: ${{ needs.target.outputs.target_branch }} + + - uses: viash-io/viash-actions/setup@v6 + + - name: Login to container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image + run: | + ${{matrix.component.executable}} ---engine docker ---setup build ---verbose + + - name: Push image + run: | + ${{matrix.component.executable}} ---engine docker ---setup push ---verbose diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml deleted file mode 100644 index 2772dc6..0000000 --- a/.github/workflows/main-build.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: main build -concurrency: main_build - -on: - push: - branches: [ 'main' ] - -jobs: - # phase 1 - list: - runs-on: ubuntu-latest - - outputs: - component_matrix: ${{ steps.set_matrix.outputs.matrix }} - cache_key: ${{ steps.cache.outputs.cache_key }} - - steps: - - uses: actions/checkout@v3 - - - uses: viash-io/viash-actions/setup@v3 - - - name: Remove target folder from .gitignore - run: | - # allow publishing the target folder - sed -i '/^target.*/d' .gitignore - - - uses: viash-io/viash-actions/ns-build@v3 - with: - config_mod: .functionality.version := 'main_build' - parallel: true - - - name: Deploy to target branch - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: . - publish_branch: main_build - - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 - with: - platform: docker - src: src - format: json - - - id: set_matrix - run: | - echo "matrix=$(jq -c '[ .[] | - { - "name": (.functionality.namespace + "/" + .functionality.name), - "config": .info.config, - "dir": .info.config | capture("^(?.*\/)").dir - } - ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT - - # phase 2 - build: - needs: list - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - component: ${{ fromJson(needs.list.outputs.component_matrix) }} - - steps: - - uses: actions/checkout@v3 - - - uses: viash-io/viash-actions/setup@v3 - - - name: Build container - uses: viash-io/viash-actions/ns-build@v3 - with: - config_mod: .functionality.version := 'main_build' - platform: docker - src: ${{ matrix.component.dir }} - setup: build - - - name: Login to container registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.GTHB_USER }} - password: ${{ secrets.GTHB_PAT }} - - - name: Push container - uses: viash-io/viash-actions/ns-build@v3 - with: - config_mod: .functionality.version := 'main_build' - platform: docker - src: ${{ matrix.component.dir }} - setup: push \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..91aa083 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,90 @@ +name: Test + +on: + push: + branches: [ main ] + pull_request: + +jobs: + # phase 1 + list: + name: List components to test + runs-on: ubuntu-latest + + outputs: + matrix: ${{ steps.ns_list_filter_nextflow.outputs.output_matrix }} + cache_key: ${{ steps.cache.outputs.cache_key }} + dest_paths: ${{ steps.cache.outputs.dest_paths }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + + - uses: viash-io/viash-actions/setup@v6 + + - uses: viash-io/viash-actions/project/sync-and-cache@v6 + id: cache + + - id: ns_list + uses: viash-io/viash-actions/ns-list@v6 + with: + format: json + runner: executable + + - id: ns_list_filter_changed + uses: viash-io/viash-actions/project/detect-changed-components@v6 + with: + input_file: "${{ steps.ns_list.outputs.output_file }}" + + - name: Filter out Nextflow scripts (testing currently not supported) + id: ns_list_filter_nextflow + run: | + OUTPUT_MATRIX=$(echo '${{ steps.ns_list_filter_changed.outputs.output_matrix }}' | jq -c 'map(select(.main_script_type != "nextflow_script"))') + echo "output_matrix=$OUTPUT_MATRIX" >> $GITHUB_OUTPUT + + # phase 2 + test: + name: Test component + needs: list + if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + component: ${{ fromJson(needs.list.outputs.matrix) }} + + steps: + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - name: Clear space + uses: data-intuitive/reclaim-the-bytes@v2 + + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 + + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + # use cache + - name: Cache resources data + if: ${{ needs.list.outputs.cache_key != '' }} + uses: actions/cache/restore@v4 + timeout-minutes: 10 + with: + path: ${{ needs.list.outputs.dest_paths }} + key: ${{ needs.list.outputs.cache_key }} + + - name: Run test + timeout-minutes: 30 + env: + VIASH_TEMP: "${{ runner.temp }}/viash_test_temp" + run: | + viash test \ + "${{ matrix.component.config }}" \ + --cpus 2 \ + --memory "16gb" diff --git a/.github/workflows/viash-test.yml b/.github/workflows/viash-test.yml deleted file mode 100644 index 9a431c2..0000000 --- a/.github/workflows/viash-test.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: viash test - -on: - pull_request: - push: - branches: [ '**' ] - -jobs: - run_ci_check_job: - runs-on: ubuntu-latest - outputs: - run_ci: ${{ steps.github_cli.outputs.check }} - steps: - - name: 'Check if branch has an existing pull request and the trigger was a push' - id: github_cli - run: | - pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url') - # If the branch has a PR and this run was triggered by a push event, do not run - if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then - echo "check=false" >> $GITHUB_OUTPUT - else - echo "check=true" >> $GITHUB_OUTPUT - fi - env: - GITHUB_TOKEN: ${{ secrets.GTHB_PAT }} - - # phase 1 - list: - needs: run_ci_check_job - env: - s3_bucket: s3://openpipelines-data/ - runs-on: ubuntu-latest - if: "needs.run_ci_check_job.outputs.run_ci == 'true'" - - outputs: - matrix: ${{ steps.set_matrix.outputs.matrix }} - cache_key: ${{ steps.cache.outputs.cache_key }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: viash-io/viash-actions/setup@v3 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v35.6.4 - with: - separator: ";" - diff_relative: true - - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 - with: - platform: docker - format: json - - - id: ns_list_filtered - uses: viash-io/viash-actions/project/detect-changed-components@v3 - with: - input_file: "${{ steps.ns_list.outputs.output_file }}" - - - id: set_matrix - run: | - echo "matrix=$(jq -c '[ .[] | - { - "name": (.functionality.namespace + "/" + .functionality.name), - "config": .info.config, - "dir": .info.config | capture("^(?.*\/)").dir - } - ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT - - # phase 2 - viash_test: - needs: list - if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }} - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - component: ${{ fromJson(needs.list.outputs.matrix) }} - - steps: - - uses: actions/checkout@v3 - - - uses: viash-io/viash-actions/setup@v3 - - - name: Run test - timeout-minutes: 30 - run: | - viash test \ - "${{ matrix.component.config }}" \ - --cpus 2 \ - --memory "5gb" -