ci: refactor on push and release workflows to use same cached build used in tests #44
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: '0 8 * * TUE' | |
| # Triggered on push by .github/workflows/release.yaml | |
| workflow_call: | |
| outputs: | |
| artifact-prefix: | |
| description: build_charm.yaml `artifact-prefix` output | |
| value: ${{ jobs.build.outputs.artifact-prefix }} | |
| charm-paths: | |
| value: ${{ jobs.get-charm-paths.outputs.charm-paths }} | |
| channel: | |
| value: ${{ jobs.get-charm-paths.outputs.channel }} | |
| jobs: | |
| lib-check: | |
| name: Check libraries | |
| strategy: | |
| matrix: | |
| charm: | |
| - kfp-api | |
| - kfp-metadata-writer | |
| - kfp-persistence | |
| - kfp-profile-controller | |
| - kfp-schedwf | |
| - kfp-ui | |
| - kfp-viewer | |
| - kfp-viz | |
| uses: canonical/charmed-kubeflow-workflows/.github/workflows/[email protected] | |
| secrets: inherit | |
| with: | |
| charm-path: ./charms/${{ matrix.charm }} | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| charm: | |
| - kfp-api | |
| - kfp-metadata-writer | |
| - kfp-persistence | |
| - kfp-profile-controller | |
| - kfp-schedwf | |
| - kfp-ui | |
| - kfp-viewer | |
| - kfp-viz | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: python3 -m pip install tox | |
| - run: tox -e ${{ matrix.charm }}-lint | |
| unit: | |
| name: Unit tests | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| charm: | |
| - kfp-api | |
| - kfp-metadata-writer | |
| - kfp-persistence | |
| - kfp-profile-controller | |
| - kfp-schedwf | |
| - kfp-ui | |
| - kfp-viewer | |
| - kfp-viz | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: python3 -m pip install tox | |
| - run: tox -e ${{ matrix.charm }}-unit | |
| terraform-checks: | |
| name: Terraform | |
| uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main | |
| strategy: | |
| matrix: | |
| charm: | |
| - kfp-api | |
| - kfp-metadata-writer | |
| - kfp-persistence | |
| - kfp-profile-controller | |
| - kfp-schedwf | |
| - kfp-ui | |
| - kfp-viewer | |
| - kfp-viz | |
| with: | |
| charm-path: ./charms/${{ matrix.charm }} | |
| # The namespace is hardcoded in the upstream project | |
| # So the model's name must be kubeflow | |
| # See: https://github.com/kubeflow/kubeflow/issues/6136 | |
| model: kubeflow | |
| get-charm-paths: | |
| name: Generate the Charm Matrix content | |
| runs-on: ubuntu-latest | |
| outputs: | |
| charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }} | |
| channel: ${{ steps.select-channel.outputs.name }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get paths for all charms in this repo | |
| id: get-charm-paths | |
| uses: canonical/kubeflow-ci/actions/get-charm-paths@main | |
| - name: Select charmhub channel | |
| uses: canonical/charming-actions/[email protected] | |
| id: select-channel | |
| build: | |
| strategy: | |
| matrix: | |
| charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm-paths) }} | |
| name: Build charm | ${{ matrix.charm }} | |
| needs: | |
| - get-charm-paths | |
| uses: canonical/data-platform-workflows/.github/workflows/[email protected] | |
| with: | |
| charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release | |
| path-to-charm-directory: ${{ matrix.charm }} | |
| release: | |
| strategy: | |
| matrix: | |
| charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm-paths) }} | |
| name: Release charm to Charmhub branch | ${{ matrix.charm }} | |
| if: ${{ github.event_name == 'pull_request' }} | |
| needs: | |
| - build | |
| uses: canonical/data-platform-workflows/.github/workflows/[email protected] | |
| with: | |
| charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release | |
| channel: ${{ needs.get-charm-paths.outputs.channel }} | |
| artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} | |
| path-to-charm-directory: ${{ matrix.charm }} | |
| create-git-tags: false | |
| secrets: | |
| charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} | |
| integration: | |
| name: Integration tests (microk8s) | |
| needs: | |
| - build | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| charm: | |
| - kfp-persistence | |
| - kfp-profile-controller | |
| - kfp-api | |
| - kfp-schedwf | |
| - kfp-viewer | |
| - kfp-ui | |
| - kfp-metadata-writer | |
| - kfp-viz | |
| steps: | |
| # Ideally we'd use self-hosted runners, but this effort is still not stable | |
| # This action will remove unused software (dotnet, haskell, android libs, codeql, | |
| # and docker images) from the GH runner. | |
| # This leaves ~45GB free as of 2024-04-10, but this amount has varied as GH changed their | |
| # runners | |
| - name: Maximise GH runner space | |
| uses: jlumbroso/[email protected] | |
| - uses: actions/checkout@v3 | |
| - name: Setup operator environment | |
| uses: charmed-kubernetes/actions-operator@main | |
| with: | |
| provider: microk8s | |
| channel: 1.29-strict/stable | |
| juju-channel: 3.6/stable | |
| charmcraft-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release | |
| - name: Download packed charm(s) | |
| id: download-charms | |
| timeout-minutes: 5 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: ${{ needs.build.outputs.artifact-prefix }}-* | |
| merge-multiple: true | |
| - name: Integration tests | |
| run: | | |
| # Requires the model to be called kubeflow due to | |
| # https://github.com/canonical/kfp-operators/issues/389 | |
| juju add-model kubeflow | |
| # Pass the path where the charm artefact is downloaded to the tox command | |
| # FIXME: Right now the complete path is half hardcoded to <charm name>[email protected] | |
| # We need to find a better way to dynamically get this value | |
| sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}[email protected]" | |
| - name: Collect charm debug artifacts | |
| uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main | |
| if: always() | |
| test-bundle: | |
| name: Test the bundle | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| sdk: | |
| - v1 | |
| - v2 | |
| steps: | |
| # This is a workaround for https://github.com/canonical/kfp-operators/issues/250 | |
| # Ideally we'd use self-hosted runners, but this effort is still not stable | |
| # This action will remove unused software (dotnet, haskell, android libs, codeql, | |
| # and docker images) from the GH runner. | |
| # This leaves ~45GB free as of 2024-04-10, but this amount has varied as GH changed their | |
| # runners | |
| - name: Maximise GH runner space | |
| uses: jlumbroso/[email protected] | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| - name: Setup operator environment | |
| uses: charmed-kubernetes/actions-operator@main | |
| with: | |
| provider: microk8s | |
| channel: 1.29-strict/stable | |
| juju-channel: 3.6/stable | |
| charmcraft-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release | |
| microk8s-addons: "dns hostpath-storage rbac metallb:10.64.140.43-10.64.140.49" | |
| - name: Run test | |
| run: | | |
| # Requires the model to be called kubeflow due to kfp-viewer | |
| juju add-model kubeflow | |
| sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }} -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2" --charmcraft-clean | |
| - name: Get all | |
| run: kubectl get all -A | |
| if: failure() | |
| - name: Get juju status | |
| run: juju status | |
| if: failure() | |
| # Collect debug artefacts only on failure || cancelled as the CI for this repository | |
| # in particular struggles with storage limitations. | |
| - name: Collect charm debug artifacts | |
| uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main | |
| if: failure() || cancelled() |