From cd02ca3d5d25cce696edc071db1fca052af004c4 Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Thu, 9 Jan 2025 14:05:27 +0100 Subject: [PATCH 1/4] ci, tests: pass charm artefacts to deploy and test charms (#640) * ci, tests: pass charm artefacts to deploy and test charms This commit enables the "--charm-path" option to pass .charm artefacts to be deployed and tested at an individual level. This change enables the option to pass pre-built charms to the tests to avoid building them on the same test. It also ensures compatibility with the build_charm.py reusable workflow (from canonical/data-platform-workflows). Fixes: #639 --- .github/workflows/integrate.yaml | 13 +++++++++- charms/kfp-api/tests/integration/conftest.py | 12 ++++++++++ .../kfp-api/tests/integration/test_charm.py | 13 ++++++---- .../tests/integration/conftest.py | 12 ++++++++++ .../tests/integration/test_charm.py | 18 ++++++++++---- .../tests/integration/conftest.py | 12 ++++++++++ .../tests/integration/test_charm.py | 14 +++++++---- .../tests/integration/conftest.py | 12 ++++++++++ .../tests/integration/test_charm.py | 24 ++++++++++++------- .../kfp-schedwf/tests/integration/conftest.py | 12 ++++++++++ .../tests/integration/test_charm.py | 17 +++++++++---- charms/kfp-ui/tests/integration/conftest.py | 12 ++++++++++ charms/kfp-ui/tests/integration/test_charm.py | 18 ++++++++++---- .../kfp-viewer/tests/integration/conftest.py | 12 ++++++++++ .../tests/integration/test_charm.py | 18 ++++++++++---- charms/kfp-viz/tests/integration/conftest.py | 12 ++++++++++ .../kfp-viz/tests/integration/test_charm.py | 19 ++++++++++----- 17 files changed, 206 insertions(+), 44 deletions(-) create mode 100644 charms/kfp-api/tests/integration/conftest.py create mode 100644 charms/kfp-metadata-writer/tests/integration/conftest.py create mode 100644 charms/kfp-persistence/tests/integration/conftest.py create mode 100644 charms/kfp-profile-controller/tests/integration/conftest.py create mode 100644 charms/kfp-schedwf/tests/integration/conftest.py create mode 100644 charms/kfp-ui/tests/integration/conftest.py create mode 100644 charms/kfp-viewer/tests/integration/conftest.py create mode 100644 charms/kfp-viz/tests/integration/conftest.py diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 17f5fc26..45ca402d 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -122,12 +122,23 @@ jobs: # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845 charmcraft-channel: 3.x/stable + - name: Download packed charm(s) + id: download-charms + timeout-minutes: 5 + uses: actions/download-artifact@v4 + with: + pattern: packed-charm-cache-true-.-charms-${{ matrix.charm }}-* + merge-multiple: true + - name: Integration tests run: | # Requires the model to be called kubeflow due to # https://github.com/canonical/kfp-operators/issues/389 juju add-model kubeflow - sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow" + # Pass the path where the charm artefact is downloaded to the tox command + # FIXME: Right now the complete path is half hardcoded to _ubuntu-20.04-amd64.charm + # We need to find a better way to dynamically get this value + sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu-20.04-amd64.charm" - name: Collect charm debug artifacts uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main diff --git a/charms/kfp-api/tests/integration/conftest.py b/charms/kfp-api/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-api/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-api/tests/integration/test_charm.py b/charms/kfp-api/tests/integration/test_charm.py index 22e4374d..a89fd834 100644 --- a/charms/kfp-api/tests/integration/test_charm.py +++ b/charms/kfp-api/tests/integration/test_charm.py @@ -43,16 +43,21 @@ class TestCharm: """Integration test charm""" @pytest.mark.abort_on_fail - async def test_build_and_deploy(self, ops_test: OpsTest): + async def test_build_and_deploy(self, ops_test: OpsTest, request): """Deploy kfp-api with required charms and relations.""" - built_charm_path = await ops_test.build_charm("./") - logger.info(f"Built charm {built_charm_path}") image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, + entity_url=entity_url, application_name=APP_NAME, resources=resources, trust=True, diff --git a/charms/kfp-metadata-writer/tests/integration/conftest.py b/charms/kfp-metadata-writer/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-metadata-writer/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-metadata-writer/tests/integration/test_charm.py b/charms/kfp-metadata-writer/tests/integration/test_charm.py index 0f0d5d77..4b4b6a13 100644 --- a/charms/kfp-metadata-writer/tests/integration/test_charm.py +++ b/charms/kfp-metadata-writer/tests/integration/test_charm.py @@ -26,16 +26,24 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy_with_relations(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm(CHARM_ROOT) - log.info(f"Built charm {built_charm_path}") - +async def test_build_and_deploy_with_relations(ops_test: OpsTest, request): image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, application_name=APP_NAME, resources=resources, trust=True + entity_url=entity_url, + application_name=APP_NAME, + resources=resources, + trust=True, ) + await ops_test.model.deploy(entity_url=MLMD, channel=MLMD_CHANNEL, trust=True) await ops_test.model.integrate(f"{MLMD}:grpc", f"{APP_NAME}:grpc") await ops_test.model.wait_for_idle(apps=[APP_NAME, MLMD], status="active", timeout=10 * 60) diff --git a/charms/kfp-persistence/tests/integration/conftest.py b/charms/kfp-persistence/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-persistence/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-persistence/tests/integration/test_charm.py b/charms/kfp-persistence/tests/integration/test_charm.py index bd0c3882..58ac45a5 100644 --- a/charms/kfp-persistence/tests/integration/test_charm.py +++ b/charms/kfp-persistence/tests/integration/test_charm.py @@ -39,16 +39,20 @@ class TestCharm: """Integration test charm""" @pytest.mark.abort_on_fail - async def test_build_and_deploy(self, ops_test: OpsTest): + async def test_build_and_deploy(self, ops_test: OpsTest, request): """Deploy kfp-persistence with required charms and relations.""" - built_charm_path = await ops_test.build_charm("./") - logger.info(f"Built charm {built_charm_path}") - image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, + entity_url=entity_url, application_name=APP_NAME, resources=resources, trust=True, diff --git a/charms/kfp-profile-controller/tests/integration/conftest.py b/charms/kfp-profile-controller/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-profile-controller/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-profile-controller/tests/integration/test_charm.py b/charms/kfp-profile-controller/tests/integration/test_charm.py index 1160bdbf..ba42f476 100644 --- a/charms/kfp-profile-controller/tests/integration/test_charm.py +++ b/charms/kfp-profile-controller/tests/integration/test_charm.py @@ -48,19 +48,14 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm("./") - logger.info(f"Built charm {built_charm_path}") - - image_path = METADATA["resources"]["oci-image"]["upstream-source"] - resources = {"oci-image": image_path} - +async def test_build_and_deploy(ops_test: OpsTest, request): # Deploy the admission webhook to apply the PodDefault CRD required by the charm workload await ops_test.model.deploy( entity_url=ADMISSION_WEBHOOK, channel=ADMISSION_WEBHOOK_CHANNEL, trust=ADMISSION_WEBHOOK_TRUST, ) + # TODO: The webhook charm must be active before the metacontroller is deployed, due to the bug # described here: https://github.com/canonical/metacontroller-operator/issues/86 # Drop this wait_for_idle once the above issue is closed @@ -72,8 +67,21 @@ async def test_build_and_deploy(ops_test: OpsTest): trust=METACONTROLLER_TRUST, ) + # Deploy the charm under test + image_path = METADATA["resources"]["oci-image"]["upstream-source"] + resources = {"oci-image": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) + await ops_test.model.deploy( - built_charm_path, application_name=CHARM_NAME, resources=resources, trust=True + entity_url=entity_url, + resources=resources, + trust=True, ) # Deploy required relations diff --git a/charms/kfp-schedwf/tests/integration/conftest.py b/charms/kfp-schedwf/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-schedwf/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-schedwf/tests/integration/test_charm.py b/charms/kfp-schedwf/tests/integration/test_charm.py index 64d8c229..4e3563b3 100644 --- a/charms/kfp-schedwf/tests/integration/test_charm.py +++ b/charms/kfp-schedwf/tests/integration/test_charm.py @@ -21,15 +21,22 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy_with_relations(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm(CHARM_ROOT) - log.info(f"Built charm {built_charm_path}") - +async def test_build_and_deploy_with_relations(ops_test: OpsTest, request): image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, application_name=APP_NAME, resources=resources, trust=True + entity_url=entity_url, + application_name=APP_NAME, + resources=resources, + trust=True, ) await ops_test.model.wait_for_idle( diff --git a/charms/kfp-ui/tests/integration/conftest.py b/charms/kfp-ui/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-ui/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-ui/tests/integration/test_charm.py b/charms/kfp-ui/tests/integration/test_charm.py index 02c59c4e..e0992c97 100644 --- a/charms/kfp-ui/tests/integration/test_charm.py +++ b/charms/kfp-ui/tests/integration/test_charm.py @@ -22,16 +22,24 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy_with_relations(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm(CHARM_ROOT) - log.info(f"Built charm {built_charm_path}") - +async def test_build_and_deploy_with_relations(ops_test: OpsTest, request): image_path = METADATA["resources"]["ml-pipeline-ui"]["upstream-source"] resources = {"ml-pipeline-ui": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, application_name=APP_NAME, resources=resources, trust=True + entity_url=entity_url, + application_name=APP_NAME, + resources=resources, + trust=True, ) + await ops_test.model.deploy(BUNDLE, trust=True) await ops_test.model.integrate(f"{APP_NAME}:kfp-api", "kfp-api:kfp-api") await ops_test.model.integrate(f"{APP_NAME}:object-storage", "minio:object-storage") diff --git a/charms/kfp-viewer/tests/integration/conftest.py b/charms/kfp-viewer/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-viewer/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-viewer/tests/integration/test_charm.py b/charms/kfp-viewer/tests/integration/test_charm.py index c96f8794..1cd1ea96 100644 --- a/charms/kfp-viewer/tests/integration/test_charm.py +++ b/charms/kfp-viewer/tests/integration/test_charm.py @@ -21,16 +21,24 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy_with_relations(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm(CHARM_ROOT) - log.info(f"Built charm {built_charm_path}") - +async def test_build_and_deploy_with_relations(ops_test: OpsTest, request): image_path = METADATA["resources"]["kfp-viewer-image"]["upstream-source"] resources = {"kfp-viewer-image": image_path} + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, application_name=APP_NAME, resources=resources, trust=True + entity_url=entity_url, + application_name=APP_NAME, + resources=resources, + trust=True, ) + await ops_test.model.wait_for_idle( apps=[APP_NAME], status="active", diff --git a/charms/kfp-viz/tests/integration/conftest.py b/charms/kfp-viz/tests/integration/conftest.py new file mode 100644 index 00000000..f4eda080 --- /dev/null +++ b/charms/kfp-viz/tests/integration/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/charms/kfp-viz/tests/integration/test_charm.py b/charms/kfp-viz/tests/integration/test_charm.py index 7eb9b68f..b712d40c 100644 --- a/charms/kfp-viz/tests/integration/test_charm.py +++ b/charms/kfp-viz/tests/integration/test_charm.py @@ -21,16 +21,23 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy_with_relations(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm(CHARM_ROOT) - log.info(f"Built charm {built_charm_path}") - +async def test_build_and_deploy_with_relations(ops_test: OpsTest, request): image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} - + # Keep the option to run the integration tests locally + # by building the charm and then deploying + entity_url = ( + await ops_test.build_charm("./") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) await ops_test.model.deploy( - entity_url=built_charm_path, application_name=APP_NAME, resources=resources, trust=True + entity_url=entity_url, + application_name=APP_NAME, + resources=resources, + trust=True, ) + await ops_test.model.wait_for_idle( apps=[APP_NAME], status="active", From e4de97af5b350515773bcab0b6492e714519e704 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Jan 2025 12:52:34 +0100 Subject: [PATCH 2/4] Use single (cached) build for integration tests & release --- .github/workflows/{integrate.yaml => ci.yaml} | 75 +++++++++++++-- .github/workflows/get-charm-paths.sh | 30 ------ .github/workflows/on_pull_request.yaml | 21 ---- .github/workflows/on_push.yaml | 28 ------ .github/workflows/promote.yaml | 32 +++++++ .github/workflows/publish.yaml | 95 ------------------- .github/workflows/release.yaml | 56 ++++++----- .github/workflows/weekly_ci.yaml | 13 --- 8 files changed, 131 insertions(+), 219 deletions(-) rename .github/workflows/{integrate.yaml => ci.yaml} (72%) delete mode 100644 .github/workflows/get-charm-paths.sh delete mode 100644 .github/workflows/on_pull_request.yaml delete mode 100644 .github/workflows/on_push.yaml create mode 100644 .github/workflows/promote.yaml delete mode 100644 .github/workflows/publish.yaml delete mode 100644 .github/workflows/weekly_ci.yaml diff --git a/.github/workflows/integrate.yaml b/.github/workflows/ci.yaml similarity index 72% rename from .github/workflows/integrate.yaml rename to .github/workflows/ci.yaml index 45ca402d..4de66c19 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/ci.yaml @@ -1,11 +1,23 @@ -# reusable workflow triggered by other actions -name: CI +name: Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true on: + pull_request: + schedule: + - cron: '0 8 * * TUE' + # Triggered on push by .github/workflows/release.yaml workflow_call: - secrets: - CHARMCRAFT_CREDENTIALS: - required: true + outputs: + artifact-prefix: + description: build_charm.yaml `artifact-prefix` output + value: ${{ jobs.build.outputs.artifact-prefix }} + charm-paths: + value: ${{ jobs.get-charm-paths.outputs.charm-paths }} + channel: + value ${{ jobs.get-charm-paths.outputs.channel }} jobs: lib-check: @@ -86,9 +98,58 @@ jobs: # So the model's name must be kubeflow # See: https://github.com/kubeflow/kubeflow/issues/6136 model: kubeflow - + + get-charm-paths: + name: Generate the Charm Matrix content + runs-on: ubuntu-latest + outputs: + charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }} + channel: ${{ steps.select-channel.outputs.name }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get paths for all charms in this repo + id: get-charm-paths + uses: canonical/kubeflow-ci/actions/get-charm-paths@main + - name: Select charmhub channel + uses: canonical/charming-actions/channel@2.6.2 + id: select-channel + + build: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm-paths) }} + name: Build charm | ${{ matrix.charm }} + needs: + - get-charm-paths + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.0 + with: + charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release + path-to-charm-directory: ${{ matrix.charm }} + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm-paths) }} + name: Release charm to Charmhub branch | ${{ matrix.charm }} + if: ${{ github.event_name == 'pull_request' }} + needs: + - build + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v29.0.0 + with: + charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release + channel: ${{ needs.get-charm-paths.outputs.channel }} + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + create-git-tags: false + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + integration: name: Integration tests (microk8s) + needs: + - build runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -127,7 +188,7 @@ jobs: timeout-minutes: 5 uses: actions/download-artifact@v4 with: - pattern: packed-charm-cache-true-.-charms-${{ matrix.charm }}-* + pattern: ${{ needs.build.outputs.artifact-prefix }}-* merge-multiple: true - name: Integration tests diff --git a/.github/workflows/get-charm-paths.sh b/.github/workflows/get-charm-paths.sh deleted file mode 100644 index f3d0cb44..00000000 --- a/.github/workflows/get-charm-paths.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -x - -# Finds the charms in this repo, outputing them as JSON -# Will return one of: -# * the relative paths of the directories listed in `./charms`, if that directory exists -# * "./", if the root directory has a "metadata.yaml" file -# * otherwise, error -# -# Modifed from: https://stackoverflow.com/questions/63517732/github-actions-build-matrix-for-lambda-functions/63736071#63736071 -CHARMS_DIR="./charms" -if [ -d "$CHARMS_DIR" ]; -then - CHARM_PATHS=$(find $CHARMS_DIR -maxdepth 1 -type d -not -path '*/\.*' -not -path "$CHARMS_DIR") -else - if [ -f "./metadata.yaml" ] - then - CHARM_PATHS="./" - else - echo "Cannot find valid charm directories - aborting" - exit 1 - fi -fi - -# Convert output to JSON string format -# { charm_paths: [...] } -CHARM_PATHS_LIST=$(echo "$CHARM_PATHS" | jq -c --slurp --raw-input 'split("\n")[:-1]') - -echo "Found CHARM_PATHS_LIST: $CHARM_PATHS_LIST" - -echo "::set-output name=CHARM_PATHS_LIST::$CHARM_PATHS_LIST" diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml deleted file mode 100644 index 3d7eef10..00000000 --- a/.github/workflows/on_pull_request.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: On Pull Request - -# On pull_request, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests - -on: - pull_request: - -jobs: - - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in parallel with tests, as we always publish in this situation - publish-charm: - name: Publish Charm - uses: ./.github/workflows/publish.yaml - secrets: inherit diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml deleted file mode 100644 index 31c42da3..00000000 --- a/.github/workflows/on_push.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: On Push - -# On push to a "special" branch, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests -# where a "special" branch is one of main or track/**, as -# by convention these branches are the source for a corresponding -# charmhub edge channel. - -on: - push: - branches: - - main - - track/** - -jobs: - - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in series with tests, and only publishes if tests passes - publish-charm: - name: Publish Charm - needs: tests - uses: ./.github/workflows/publish.yaml - secrets: inherit diff --git a/.github/workflows/promote.yaml b/.github/workflows/promote.yaml new file mode 100644 index 00000000..8a33b385 --- /dev/null +++ b/.github/workflows/promote.yaml @@ -0,0 +1,32 @@ +# reusable workflow triggered manually +name: Promote charm to other tracks and channels + +on: + workflow_dispatch: + inputs: + destination-channel: + description: 'Destination Channel' + required: true + origin-channel: + description: 'Origin Channel' + required: true + charm-name: + description: 'Charm subdirectory name' + required: true + +jobs: + promote-charm: + name: Promote charm + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Release charm to channel + # TODO: use canonical/charming-actions/promote-charm? + uses: canonical/charming-actions/release-charm@2.6.2 + with: + credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + github-token: ${{ secrets.GITHUB_TOKEN }} + destination-channel: ${{ github.event.inputs.destination-channel }} + origin-channel: ${{ github.event.inputs.origin-channel }} + tag-prefix: ${{ github.event.inputs.charm-name }} + charm-path: charms/${{ github.event.inputs.charm-name}} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 33b09f50..00000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# reusable workflow for publishing all charms in this repo -name: Publish - -on: - workflow_call: - inputs: - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - default: '' - required: false - type: string - secrets: - CHARMCRAFT_CREDENTIALS: - required: true - workflow_dispatch: - inputs: - destination_channel: - description: CharmHub channel to publish to - required: false - default: 'latest/edge' - type: string - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - required: false - default: '' - type: string - -jobs: - get-charm-paths: - name: Generate the Charm Matrix - runs-on: ubuntu-20.04 - outputs: - charm_paths_list: ${{ steps.get-charm-paths.outputs.CHARM_PATHS_LIST }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - name: Get paths for all charms in repo - id: get-charm-paths - run: bash .github/workflows/get-charm-paths.sh - - - publish-charm: - name: Publish Charm - runs-on: ubuntu-20.04 - needs: get-charm-paths - strategy: - fail-fast: false - matrix: - charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - - name: Select charmhub channel - uses: canonical/charming-actions/channel@2.6.2 - id: select-channel - if: ${{ inputs.destination_channel == '' }} - - # Combine inputs from different sources to a single canonical value so later steps don't - # need logic for picking the right one - - name: Parse and combine inputs - id: parse-inputs - run: | - # destination_channel - destination_channel="${{ inputs.destination_channel || steps.select-channel.outputs.name }}" - echo "setting output of destination_channel=$destination_channel" - echo "::set-output name=destination_channel::$destination_channel" - - # tag_prefix - # if charm_path = ./ --> tag_prefix = '' (null) - # if charm_path != ./some-charm (eg: a charm in a ./charms dir) --> tag_prefix = 'some-charm' - if [ ${{ matrix.charm-path }} == './' ]; then - tag_prefix='' - else - tag_prefix=$(basename ${{ matrix.charm-path }} ) - fi - echo "setting output of tag_prefix=$tag_prefix" - echo "::set-output name=tag_prefix::$tag_prefix" - - - name: Upload charm to charmhub - uses: canonical/charming-actions/upload-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - charm-path: ${{ matrix.charm-path }} - channel: ${{ steps.parse-inputs.outputs.destination_channel }} - tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }} - # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845 - charmcraft-channel: 3.x/stable diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 57fc88dc..0d90bb39 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,31 +1,37 @@ -# reusable workflow triggered manually -name: Release charm to other tracks and channels +name: Release to Charmhub on: + push: + branches: + - main + - track/** workflow_dispatch: inputs: - destination-channel: - description: 'Destination Channel' - required: true - origin-channel: - description: 'Origin Channel' - required: true - charm-name: - description: 'Charm subdirectory name' - required: true + destination_channel: + description: CharmHub channel to publish to + required: false + default: 'latest/edge' + type: string jobs: - promote-charm: - name: Promote charm - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - name: Release charm to channel - uses: canonical/charming-actions/release-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - destination-channel: ${{ github.event.inputs.destination-channel }} - origin-channel: ${{ github.event.inputs.origin-channel }} - tag-prefix: ${{ github.event.inputs.charm-name }} - charm-path: charms/${{ github.event.inputs.charm-name}} + ci-tests: + uses: ./.github/workflows/ci.yaml + secrets: inherit + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.ci-tests.outputs.charm-paths) }} + name: Release charm | ${{ matrix.charm }} + needs: + - ci-tests + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v29.0.0 + with: + charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release + channel: ${{ inputs.destination_channel || needs.ci-tests.outputs.channel }} + artifact-prefix: ${{ needs.ci-tests.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + contents: write # Needed to create git tags diff --git a/.github/workflows/weekly_ci.yaml b/.github/workflows/weekly_ci.yaml deleted file mode 100644 index ef427c1e..00000000 --- a/.github/workflows/weekly_ci.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Run weekly tests - -on: - schedule: - - cron: '0 8 * * TUE' - -jobs: - - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: - charmcraft-credentials: "${{ secrets.CHARMCRAFT_CREDENTIALS }}" From cdae892475e0e2608ce96cfdd4bb85108782b992 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Jan 2025 12:57:27 +0100 Subject: [PATCH 3/4] Fix hardcoded path for `platforms` syntax --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4de66c19..1cd04fd0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -197,9 +197,9 @@ jobs: # https://github.com/canonical/kfp-operators/issues/389 juju add-model kubeflow # Pass the path where the charm artefact is downloaded to the tox command - # FIXME: Right now the complete path is half hardcoded to _ubuntu-20.04-amd64.charm + # FIXME: Right now the complete path is half hardcoded to _ubuntu@20.04-amd64.charm # We need to find a better way to dynamically get this value - sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu-20.04-amd64.charm" + sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu@20.04-amd64.charm" - name: Collect charm debug artifacts uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main From abcecfee1c6786db08c6e542779a5bab18fcd7b0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Jan 2025 13:01:12 +0100 Subject: [PATCH 4/4] fixup! Use single (cached) build for integration tests & release --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1cd04fd0..f0cfa333 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ on: charm-paths: value: ${{ jobs.get-charm-paths.outputs.charm-paths }} channel: - value ${{ jobs.get-charm-paths.outputs.channel }} + value: ${{ jobs.get-charm-paths.outputs.channel }} jobs: lib-check: