update #33
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.inputs.deploy_branch && format('{0}_build', github.ref_name) || github.event.inputs.deploy_branch }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| push_containers: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Build docker images and push them to the registry | |
| version: | |
| type: string | |
| required: false | |
| description: | | |
| Version to tag the build components with (e.i functionality.version). | |
| Defaults to name of the branch that triggered the workflow, suffixed by "_build". | |
| target_tag: | |
| type: string | |
| required: false | |
| default: main_build | |
| description: | | |
| Version tag of containers to use. Is `main_build` by default. | |
| Can be used in combination with 'push_containers' to re-use existing docker images or set the tag for new builds. | |
| deploy_to_viash_hub: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Also build packages and docker images for viash-hub.com and push them. | |
| # when used as a subworkflow | |
| workflow_call: | |
| inputs: | |
| push_containers: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: push the containers to the registry | |
| version: | |
| type: string | |
| required: false | |
| description: | | |
| Version to tag the build components with (e.i functionality.version). | |
| Defaults to name of the branch that triggered the workflow, suffixed by "_build". | |
| target_tag: | |
| type: string | |
| required: false | |
| default: main_build | |
| description: Version tag of existing containers to use. Is `main_build` by default. | |
| deploy_branch: | |
| type: string | |
| required: false | |
| description: | | |
| Branch to deploy the build to. Defaults to name of the branch | |
| that triggered the workflow, suffixed by "_build". | |
| deploy_to_viash_hub: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Also build packages and docker images for viash-hub.com and push them. | |
| outputs: | |
| component_matrix: | |
| description: "A JSON object that can be used to populate a github actions matrix for component jobs." | |
| value: ${{ jobs.build_and_deploy_target_folder.outputs.component_matrix }} | |
| workflow_matrix: | |
| description: "A JSON object that can be used to populate a github actions matrix for workflow jobs." | |
| value: ${{ jobs.build_and_deploy_target_folder.outputs.workflow_matrix}} | |
| secrets: | |
| VIASHHUB_USER: | |
| required: true | |
| VIASH_HUB_RNASEQ: | |
| required: true | |
| # GTHB_USER: | |
| # required: true | |
| # GTHB_PAT: | |
| # required: true | |
| push: | |
| branches: [ 'main' ] | |
| jobs: | |
| # phase 1 | |
| build_and_deploy_target_folder: | |
| name: "Build and push target folder" | |
| runs-on: ubuntu-latest | |
| outputs: | |
| component_matrix: ${{ steps.set_matrix.outputs.components }} | |
| workflow_matrix: ${{ steps.set_matrix.outputs.workflows }} | |
| env: | |
| DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| steps: | |
| - name: Keep symlinks as-is | |
| run: | | |
| git config --global core.symlinks true | |
| - uses: actions/checkout@v4 | |
| if: ${{ inputs.deploy_to_viash_hub == 'true' }} | |
| with: | |
| fetch-depth: 0 | |
| - name: Push ref to Viash-hub | |
| if: ${{ inputs.deploy_to_viash_hub == 'true' }} | |
| run: | | |
| git remote add viash-hub https://${{ secrets.VIASH_HUB_RNASEQ }}@https://viash-hub.com/data-intuitive/rnaseq.vsh.git | |
| git push -f -u viash-hub ${{ github.ref_name }} | |
| - name: Branch to checkout (use existing target branch if it exists) | |
| id: get_checkout_branch | |
| run: | | |
| if ! git ls-remote --heads --exit-code https://github.com/data-intuitive/rnaseq.vsh.git "$DEPLOY_BRANCH" > /dev/null; then | |
| echo "Remote branch does not exist, fetching current branch and building on top of it" | |
| echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Remote branch exists, checking out existing branch" | |
| echo "checkout_branch=$DEPLOY_BRANCH" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }} | |
| fetch-depth: 0 | |
| - name: Fetch changes from ${{github.ref_name}} | |
| run: | | |
| git fetch origin ${{github.ref_name}} | |
| git checkout -f --no-overlay origin/${{github.ref_name}} -- '.' | |
| - uses: viash-io/viash-actions/setup@v4 | |
| - name: Remove target folder from .gitignore | |
| run: | | |
| # allow publishing the target folder | |
| sed -i '/^target$/d' .gitignore | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: | | |
| .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}' | |
| parallel: true | |
| query: ^(?!workflows) | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| parallel: true | |
| query: ^workflows | |
| # - name: Build nextflow schemas | |
| # uses: viash-io/viash-actions/pro/build-nextflow-schemas@v4 | |
| # with: | |
| # components: src | |
| # workflows: src | |
| # viash_pro_token: ${{ secrets.GTHB_PAT }} | |
| # tools_version: 'main_build' | |
| # - name: Build parameter files | |
| # uses: viash-io/viash-actions/pro/build-nextflow-params@v4 | |
| # with: | |
| # workflows: src | |
| # components: src | |
| # viash_pro_token: ${{ secrets.GTHB_PAT }} | |
| # tools_version: 'main_build' | |
| - name: Deploy to target branch | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| create_branch: true | |
| commit_message: "deploy: ${{github.sha}}" | |
| skip_dirty_check: true | |
| branch: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| - name: "List components" | |
| id: ns_list | |
| uses: viash-io/viash-actions/ns-list@v4 | |
| with: | |
| platform: docker | |
| src: src | |
| format: json | |
| query_namespace: ^(?!workflows) | |
| - name: "List workflows" | |
| id: ns_list_workflows | |
| uses: viash-io/viash-actions/ns-list@v4 | |
| with: | |
| src: src | |
| format: json | |
| query_namespace: ^workflows | |
| - name: "Parse JSON output from 'viash ns list' as input for matrix." | |
| id: set_matrix | |
| run: | | |
| echo "components=$(jq -c '[ .[] | | |
| { | |
| "name": (.functionality.namespace + "/" + .functionality.name), | |
| "config": .info.config, | |
| "dir": .info.config | capture("^(?<dir>.*\/)").dir | |
| } | |
| ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT | |
| echo "workflows=$(jq -c '[ .[] | . as $config | (.functionality.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) | | |
| { | |
| "name": ($config.functionality.namespace + "/" + $config.functionality.name), | |
| "main_script": (($config.info.config | capture("^(?<dir>.*\/)").dir) + "/" + .path), | |
| "entry": .entrypoint, | |
| "config": $config.info.config | |
| } | |
| ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT | |
| - uses: actions/checkout@v4 | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| with: | |
| ref: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| fetch-depth: 0 | |
| clean: true | |
| - name: Set origin to viash-hub and commit on top of it. | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| # This is needed because git-auto-commit-action uses origin by default | |
| run: | | |
| git remote add viash-hub https://x-access-token:${{ secrets.VIASH_HUB_RNASEQ }}@https://viash-hub.com/data-intuitive/rnaseq.vsh.git | |
| if git ls-remote --heads --exit-code https://viash-hub.com/data-intuitive/rnaseq.vsh.git ${{ github.ref_name }}_build > /dev/null; then | |
| git fetch viash-hub ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| git reset --hard viash-hub/${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| fi | |
| git checkout -f --no-overlay origin/${{github.ref_name}} -- '.' | |
| git remote set-url origin https://${{ secrets.VIASH_HUB_RNASEQ }}@https://viash-hub.com/data-intuitive/rnaseq.vsh.git | |
| git remote rm viash-hub | |
| - name: Remove target folder from .gitignore | |
| run: | | |
| # allow publishing the target folder | |
| sed -i '/^target$/d' .gitignore | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| with: | |
| config_mod: | | |
| .functionality.version := " ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }}" | |
| .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}' | |
| .platforms[.type == 'docker'].target_organization := 'data-intuitive' | |
| .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050' | |
| .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/data-intuitive/rnaseq.vsh' | |
| parallel: true | |
| query: ^(?!workflows) | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| with: | |
| config_mod: | | |
| .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| parallel: true | |
| query: ^workflows | |
| - name: Deploy to target branch | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| create_branch: true | |
| commit_message: "deploy: ${{github.sha}}" | |
| skip_dirty_check: true | |
| branch: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} | |
| skip_checkout: true | |
| # phase 2 | |
| build_and_deploy_docker_containers: | |
| name: "Build and Deploy Docker Images" | |
| needs: build_and_deploy_target_folder | |
| if: ${{github.event_name == 'push' || inputs.push_containers }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| component: ${{ fromJson(needs.build_and_deploy_target_folder.outputs.component_matrix) }} | |
| steps: | |
| # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' | |
| - uses: data-intuitive/reclaim-the-bytes@v2 | |
| - uses: actions/checkout@v4 | |
| - uses: viash-io/viash-actions/setup@v4 | |
| - name: Build container | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: | | |
| .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050' | |
| .platforms[.type == 'docker'].target_organization := 'data-intuitive' | |
| .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/data-intuitive/rnaseq.vsh' | |
| .platforms[.type == 'docker'].target_tag := '${{ github.ref_name }}_build' | |
| platform: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: build | |
| - name: Login to Viash-Hub container registry | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: viash-hub.com:5050 | |
| username: ${{ secrets.VIASHHUB_USER }} | |
| password: ${{ secrets.VIASH_HUB_RNASEQ }} | |
| - name: Update Docker settings | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| run: | | |
| sudo sed -i 's/ }/, \"max-concurrent-uploads\": 2 }/' /etc/docker/daemon.json | |
| sudo systemctl restart docker | |
| - name: Push container to Viash-Hub | |
| if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }} | |
| uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: | | |
| .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" | |
| .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050' | |
| .platforms[.type == 'docker'].target_organization := 'data-intuitive' | |
| .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/data-intuitive/rnaseq.vsh' | |
| platform: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: push |