main-build.yml

name: Build
concurrency: 
  group: ${{ github.workflow }}-${{ github.event.inputs.deploy_branch && format('{0}_build', github.ref_name) || github.event.inputs.deploy_branch }}
  cancel-in-progress: true

on:
  workflow_dispatch:
    inputs:
      push_containers:
        type: boolean
        required: false
        default: false
        description: Build docker images and push them to the registry
      version:
        type: string
        required: false
        description: |
          Version to tag the build components with (e.i functionality.version).
          Defaults to name of the branch that triggered the workflow, suffixed by "_build".
      target_tag:
        type: string
        required: false
        default: main_build
        description: |
          Version tag of containers to use. Is `main_build` by default.
          Can be used in combination with 'push_containers' to re-use existing docker images or set the tag for new builds.
      deploy_to_viash_hub:
        type: boolean
        required: false
        default: false
        description: Also build packages and docker images for viash-hub.com and push them.

  # when used as a subworkflow
  workflow_call:
    inputs:
      push_containers:
        type: boolean
        required: false
        default: false
        description: push the containers to the registry
      version:
        type: string
        required: false
        description: |
          Version to tag the build components with (e.i functionality.version).
          Defaults to name of the branch that triggered the workflow, suffixed by "_build".
      target_tag:
        type: string
        required: false
        default: main_build
        description: Version tag of existing containers to use. Is `main_build` by default.
      deploy_branch:
        type: string
        required: false
        description: |
          Branch to deploy the build to. Defaults to name of the branch 
          that triggered the workflow, suffixed by "_build".
      deploy_to_viash_hub:
        type: boolean
        required: false
        default: false
        description: Also build packages and docker images for viash-hub.com and push them.
    outputs:
      component_matrix:
        description: "A JSON object that can be used to populate a github actions matrix for component jobs."
        value: ${{ jobs.build_and_deploy_target_folder.outputs.component_matrix }}
      workflow_matrix:
        description: "A JSON object that can be used to populate a github actions matrix for workflow jobs."
        value: ${{ jobs.build_and_deploy_target_folder.outputs.workflow_matrix}}
    secrets:
      VIASHHUB_USER:
        required: true
      VIASH_HUB_RNASEQ:
        required: true
      # GTHB_USER:
      #   required: true 
      # GTHB_PAT:
      #   required: true
  push:
    branches: [ 'main' ]


jobs:
  # phase 1
  build_and_deploy_target_folder:
    name: "Build and push target folder"
    runs-on: ubuntu-latest

    outputs:
      component_matrix: ${{ steps.set_matrix.outputs.components }}
      workflow_matrix: ${{ steps.set_matrix.outputs.workflows }}

    env:
      DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}

    steps:
    - name: Keep symlinks as-is
      run: |
        git config --global core.symlinks true

    - uses: actions/checkout@v4
      if: ${{ inputs.deploy_to_viash_hub == 'true' }}
      with:
        fetch-depth: 0
    
    - name: Push ref to Viash-hub
      if: ${{ inputs.deploy_to_viash_hub == 'true' }}
      run: |
        git remote add viash-hub https://${{ secrets.VIASH_HUB_RNASEQ }}@https://viash-hub.com/data-intuitive/rnaseq.vsh.git
        git push -f -u viash-hub ${{ github.ref_name }}
  
    - name: Branch to checkout (use existing target branch if it exists)
      id: get_checkout_branch
      run: |
        if ! git ls-remote --heads --exit-code https://github.com/data-intuitive/rnaseq.vsh.git "$DEPLOY_BRANCH" > /dev/null; then
          echo "Remote branch does not exist, fetching current branch and building on top of it"
          echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
        else
          echo "Remote branch exists, checking out existing branch"
          echo "checkout_branch=$DEPLOY_BRANCH" >> "$GITHUB_OUTPUT"
        fi
  
    - uses: actions/checkout@v4
      with:
        ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }}
        fetch-depth: 0

    - name: Fetch changes from ${{github.ref_name}} 
      run: |
        git fetch origin ${{github.ref_name}} 
        git checkout -f --no-overlay origin/${{github.ref_name}} -- '.'
    
    - uses: viash-io/viash-actions/setup@v4

    - name: Remove target folder from .gitignore
      run: |
        # allow publishing the target folder
        sed -i '/^target$/d' .gitignore

    - uses: viash-io/viash-actions/ns-build@v4
      with:
        config_mod: |
          .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
        parallel: true
        query: ^(?!workflows)
    
    - uses: viash-io/viash-actions/ns-build@v4
      with:
        config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
        parallel: true
        query: ^workflows
  
    # - name: Build nextflow schemas
    #   uses: viash-io/viash-actions/pro/build-nextflow-schemas@v4
    #   with:
    #     components: src
    #     workflows: src
    #     viash_pro_token: ${{ secrets.GTHB_PAT }}
    #     tools_version: 'main_build'
   
    # - name: Build parameter files
    #   uses: viash-io/viash-actions/pro/build-nextflow-params@v4
    #   with:
    #     workflows: src
    #     components: src
    #     viash_pro_token: ${{ secrets.GTHB_PAT }}
    #     tools_version: 'main_build'

    - name: Deploy to target branch
      uses: stefanzweifel/git-auto-commit-action@v5
      with:
        create_branch: true
        commit_message: "deploy: ${{github.sha}}"
        skip_dirty_check: true
        branch:  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
    
    - name: "List components"
      id: ns_list
      uses: viash-io/viash-actions/ns-list@v4
      with:
        platform: docker
        src: src
        format: json
        query_namespace: ^(?!workflows)

    - name: "List workflows"
      id: ns_list_workflows
      uses: viash-io/viash-actions/ns-list@v4
      with:
        src: src
        format: json
        query_namespace: ^workflows

    - name: "Parse JSON output from 'viash ns list' as input for matrix."
      id: set_matrix
      run: |
        echo "components=$(jq -c '[ .[] | 
          { 
            "name": (.functionality.namespace + "/" + .functionality.name),
            "config": .info.config,
            "dir": .info.config | capture("^(?<dir>.*\/)").dir
          }
        ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT

        echo "workflows=$(jq -c '[ .[] | . as $config | (.functionality.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) |
        { 
          "name": ($config.functionality.namespace + "/" + $config.functionality.name),
          "main_script": (($config.info.config | capture("^(?<dir>.*\/)").dir) + "/" + .path),
          "entry": .entrypoint,
          "config": $config.info.config
        }
        ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT

    - uses: actions/checkout@v4
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      with:
        ref:  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
        fetch-depth: 0
        clean: true

    - name: Set origin to viash-hub and commit on top of it.
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      # This is needed because git-auto-commit-action uses origin by default
      run: |
        git remote add viash-hub https://x-access-token:${{ secrets.VIASH_HUB_RNASEQ }}@https://viash-hub.com/data-intuitive/rnaseq.vsh.git
        if git ls-remote --heads --exit-code https://viash-hub.com/data-intuitive/rnaseq.vsh.git ${{ github.ref_name }}_build > /dev/null; then
          git fetch viash-hub  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
          git reset --hard viash-hub/${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
        fi
        git checkout -f --no-overlay origin/${{github.ref_name}} -- '.'
        git remote set-url origin https://${{ secrets.VIASH_HUB_RNASEQ }}@https://viash-hub.com/data-intuitive/rnaseq.vsh.git
        git remote rm viash-hub

    - name: Remove target folder from .gitignore
      run: |
        # allow publishing the target folder
        sed -i '/^target$/d' .gitignore

    - uses: viash-io/viash-actions/ns-build@v4
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      with:
        config_mod: |
          .functionality.version := " ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}"
          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
          .platforms[.type == 'docker'].target_organization := 'data-intuitive'
          .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050'
          .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/data-intuitive/rnaseq.vsh'
        parallel: true
        query: ^(?!workflows)
      
    - uses: viash-io/viash-actions/ns-build@v4
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      with:
        config_mod: |
          .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
        parallel: true
        query: ^workflows
   
    - name: Deploy to target branch
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      uses: stefanzweifel/git-auto-commit-action@v5
      with:
        create_branch: true
        commit_message: "deploy: ${{github.sha}}"
        skip_dirty_check: true
        branch:  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
        skip_checkout: true

  # phase 2
  build_and_deploy_docker_containers:
    name: "Build and Deploy Docker Images"
    needs: build_and_deploy_target_folder
    if: ${{github.event_name == 'push' || inputs.push_containers }}

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        component: ${{ fromJson(needs.build_and_deploy_target_folder.outputs.component_matrix) }}

    steps:
    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
    - uses: data-intuitive/reclaim-the-bytes@v2

    - uses: actions/checkout@v4

    - uses: viash-io/viash-actions/setup@v4

    - name: Build container
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      uses: viash-io/viash-actions/ns-build@v4
      with:
        config_mod: |
          .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
          .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050'
          .platforms[.type == 'docker'].target_organization := 'data-intuitive'
          .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/data-intuitive/rnaseq.vsh'
          .platforms[.type == 'docker'].target_tag := '${{ github.ref_name }}_build'
        platform: docker
        src: ${{ matrix.component.dir }}
        setup: build
    
    - name: Login to Viash-Hub container registry
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      uses: docker/login-action@v3
      with:
        registry: viash-hub.com:5050
        username: ${{ secrets.VIASHHUB_USER }}
        password: ${{ secrets.VIASH_HUB_RNASEQ }}

    - name: Update Docker settings
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      run: |
        sudo sed -i 's/ }/, \"max-concurrent-uploads\": 2 }/' /etc/docker/daemon.json
        sudo systemctl restart docker
  
    - name: Push container to Viash-Hub
      if: ${{ github.event_name == 'push' || inputs.deploy_to_viash_hub }}
      uses: viash-io/viash-actions/ns-build@v4
      with:
        config_mod: |
          .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
          .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050'
          .platforms[.type == 'docker'].target_organization := 'data-intuitive'
          .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/data-intuitive/rnaseq.vsh'
        platform: docker
        src: ${{ matrix.component.dir }}
        setup: push