From 97ba3768dc47435a8a4df6c0e86b93a03af9f7ca Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Thu, 16 Jan 2025 19:01:31 +0100 Subject: [PATCH 01/13] ci: remove wrapper around get charm paths and build with cache workflows This commit removes the extra abstraction of the get-charm-paths action and build_charm.yaml reusable workflow to avoid potential incompatibilities and sync issues with the data platforms workflows. --- .github/workflows/on_pull_request.yaml | 28 +++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 9bcc2213..af963272 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -8,9 +8,31 @@ on: pull_request: jobs: - get-paths-and-build: - name: Get charm paths and build with cache - uses: canonical/charmed-kubeflow-workflows/.github/workflows/get_charms_build_with_cache.yaml@main + get-charm-paths: + name: Generate the Charm Matrix content + runs-on: ubuntu-latest + outputs: + charm_paths: ${{ steps.get-charm-paths.outputs.charm-paths }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get paths for all charms in this repo + id: get-charm-paths + uses: canonical/kubeflow-ci/actions/get-charm-paths@main + + build: + name: Build charms + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 + needs: get-charm-paths + strategy: + fail-fast: false + matrix: + charm: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths) }} + with: + cache: true + charmcraft-snap-channel: 3.x/stable + path-to-charm-directory: ./${{ matrix.charm }} tests: name: Run Tests From f9531ef741a41b3febdfbc7f5ee7c07574033c45 Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Thu, 16 Jan 2025 19:05:14 +0100 Subject: [PATCH 02/13] skip: fix needs --- .github/workflows/on_pull_request.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index af963272..0beee639 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -23,6 +23,7 @@ jobs: build: name: Build charms + needs: [get-charm-paths] uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 needs: get-charm-paths strategy: @@ -36,13 +37,13 @@ jobs: tests: name: Run Tests - needs: [get-paths-and-build] + needs: [build] uses: ./.github/workflows/integrate.yaml secrets: inherit # publish runs in parallel with tests, as we always publish in this situation publish-charm: name: Publish Charm - needs: [get-paths-and-build] + needs: [build] uses: ./.github/workflows/publish.yaml secrets: inherit From 4d8278cd14b4d1ada8654a55d80ef6515c7ee5dd Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Thu, 16 Jan 2025 19:06:18 +0100 Subject: [PATCH 03/13] skip: fix syntax error --- .github/workflows/on_pull_request.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 0beee639..0f91f9bf 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -23,7 +23,6 @@ jobs: build: name: Build charms - needs: [get-charm-paths] uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 needs: get-charm-paths strategy: @@ -37,13 +36,13 @@ jobs: tests: name: Run Tests - needs: [build] + needs: build uses: ./.github/workflows/integrate.yaml secrets: inherit # publish runs in parallel with tests, as we always publish in this situation publish-charm: name: Publish Charm - needs: [build] + needs: build uses: ./.github/workflows/publish.yaml secrets: inherit From 9648209e716ae9f20a63ed08dd86e1185879aa51 Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Fri, 17 Jan 2025 17:46:24 +0100 Subject: [PATCH 04/13] skip: update based on feedback --- .github/workflows/on_pull_request.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 0f91f9bf..3a4470ba 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -23,16 +23,15 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.5 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.1 needs: get-charm-paths strategy: - fail-fast: false matrix: charm: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths) }} with: - cache: true - charmcraft-snap-channel: 3.x/stable - path-to-charm-directory: ./${{ matrix.charm }} + # Update to 3.x/stable once the latest/candidate is released to that track + charmcraft-snap-channel: latest/candidate + path-to-charm-directory: ${{ matrix.charm }} tests: name: Run Tests From 0ec29a5e22248b81b93c0722caa76d6460b5e8ee Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Mon, 20 Jan 2025 15:16:54 +0200 Subject: [PATCH 05/13] update workflows for dpw v29 --- .github/workflows/integrate.yaml | 10 +++++++++- .github/workflows/on_pull_request.yaml | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index d1a8b17c..65b9884a 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -6,6 +6,14 @@ on: secrets: CHARMCRAFT_CREDENTIALS: required: true + inputs: + artifact-prefix: + description: | + Prefix for charm package GitHub artifact(s) + + Use canonical/data-platform-workflows build_charm.yaml to build the charm(s) + required: true + type: string jobs: lib-check: @@ -126,7 +134,7 @@ jobs: timeout-minutes: 5 uses: actions/download-artifact@v4 with: - pattern: packed-charm-cache-true-.-charms-${{ matrix.charm }}-* + pattern: ${{ inputs.artifact-prefix }}-${{ matrix.charm }}-* merge-multiple: true - name: Integration tests diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 3a4470ba..d45da2a4 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -38,6 +38,8 @@ jobs: needs: build uses: ./.github/workflows/integrate.yaml secrets: inherit + with: + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} # publish runs in parallel with tests, as we always publish in this situation publish-charm: From 01c0ae20e5edb774b7ab9f22a07a023a9f650d5a Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Mon, 20 Jan 2025 17:05:26 +0200 Subject: [PATCH 06/13] fix: use correct artifact name --- .github/workflows/integrate.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 65b9884a..45150eb5 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -134,7 +134,7 @@ jobs: timeout-minutes: 5 uses: actions/download-artifact@v4 with: - pattern: ${{ inputs.artifact-prefix }}-${{ matrix.charm }}-* + pattern: ${{ inputs.artifact-prefix }}-* merge-multiple: true - name: Integration tests @@ -145,7 +145,7 @@ jobs: # Pass the path where the charm artefact is downloaded to the tox command # FIXME: Right now the complete path is half hardcoded to _ubuntu-20.04-amd64.charm # We need to find a better way to dynamically get this value - sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu-20.04-amd64.charm" + sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu@20.04-amd64.charm" - name: Collect charm debug artifacts uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main From 65ca9c525df32400e8affc33db453207afe25f00 Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 11:00:11 +0200 Subject: [PATCH 07/13] address review --- .github/workflows/on_pull_request.yaml | 4 ++-- .github/workflows/publish.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index d45da2a4..7fba46cc 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -22,12 +22,12 @@ jobs: uses: canonical/kubeflow-ci/actions/get-charm-paths@main build: - name: Build charms + name: Build charm | ${{ matrix.charm }} uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.1 needs: get-charm-paths strategy: matrix: - charm: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths) }} + charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm_paths) }} with: # Update to 3.x/stable once the latest/candidate is released to that track charmcraft-snap-channel: latest/candidate diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 9284c033..e591d822 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -48,7 +48,7 @@ jobs: strategy: fail-fast: false matrix: - charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }} + charm-path: ${{ fromJSON(needs.get-charm-paths.outputs.charm_paths_list) }} steps: - name: Checkout From a739abcbf86475170c4cae9fc0ddc4e9a1038ecb Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 11:04:05 +0200 Subject: [PATCH 08/13] disable destructive mode in publish-charm --- .github/workflows/publish.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index e591d822..663ca7a6 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -92,3 +92,4 @@ jobs: channel: ${{ steps.parse-inputs.outputs.destination_channel }} tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }} charmcraft-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release + destructive-mode: false From 9ac761c856f381d4a6e34557f58312204a2c49d3 Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 11:33:53 +0200 Subject: [PATCH 09/13] setup lxd for packing in non-destructive mode --- .github/workflows/publish.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 663ca7a6..8b42ab65 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -83,6 +83,12 @@ jobs: echo "setting output of tag_prefix=$tag_prefix" echo "::set-output name=tag_prefix::$tag_prefix" + # Required to charmcraft pack in non-destructive mode + - name: Setup lxd + uses: canonical/setup-lxd@v0.1.2 + with: + channel: latest/stable + - name: Upload charm to charmhub uses: canonical/charming-actions/upload-charm@2.6.2 with: From 8ebf63ebc7665130660ff8465cfdd0d2b22917c4 Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 13:31:06 +0200 Subject: [PATCH 10/13] Revert "setup lxd for packing in non-destructive mode" This reverts commit 9ac761c856f381d4a6e34557f58312204a2c49d3. --- .github/workflows/publish.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 8b42ab65..663ca7a6 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -83,12 +83,6 @@ jobs: echo "setting output of tag_prefix=$tag_prefix" echo "::set-output name=tag_prefix::$tag_prefix" - # Required to charmcraft pack in non-destructive mode - - name: Setup lxd - uses: canonical/setup-lxd@v0.1.2 - with: - channel: latest/stable - - name: Upload charm to charmhub uses: canonical/charming-actions/upload-charm@2.6.2 with: From 1a65f8dea58aee40c17ac508d68eb43d7b39cc36 Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 13:31:24 +0200 Subject: [PATCH 11/13] Revert "disable destructive mode in publish-charm" This reverts commit a739abcbf86475170c4cae9fc0ddc4e9a1038ecb. --- .github/workflows/publish.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 663ca7a6..e591d822 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -92,4 +92,3 @@ jobs: channel: ${{ steps.parse-inputs.outputs.destination_channel }} tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }} charmcraft-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release - destructive-mode: false From 47b983c3857b37ec412ab9de70beca1a4be44d1f Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 13:57:53 +0200 Subject: [PATCH 12/13] nit: address comments for consistency with data repos --- .github/workflows/on_pull_request.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 7fba46cc..c138f7b8 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -12,7 +12,7 @@ jobs: name: Generate the Charm Matrix content runs-on: ubuntu-latest outputs: - charm_paths: ${{ steps.get-charm-paths.outputs.charm-paths }} + charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }} steps: - uses: actions/checkout@v4 with: @@ -22,15 +22,14 @@ jobs: uses: canonical/kubeflow-ci/actions/get-charm-paths@main build: - name: Build charm | ${{ matrix.charm }} - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.1 - needs: get-charm-paths strategy: matrix: - charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm_paths) }} + charm: ${{ fromJSON(needs.get-charm-paths.outputs.charm-paths) }} + name: Build charm | ${{ matrix.charm }} + needs: + - get-charm-paths with: - # Update to 3.x/stable once the latest/candidate is released to that track - charmcraft-snap-channel: latest/candidate + charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release path-to-charm-directory: ${{ matrix.charm }} tests: From 23ef6b94ee032ee7807eda2f64ef8be6992df274 Mon Sep 17 00:00:00 2001 From: NohaIhab Date: Tue, 21 Jan 2025 15:10:25 +0200 Subject: [PATCH 13/13] skip: fix typo --- .github/workflows/on_pull_request.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index c138f7b8..59102ae5 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -28,6 +28,7 @@ jobs: name: Build charm | ${{ matrix.charm }} needs: - get-charm-paths + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.0 with: charmcraft-snap-channel: latest/candidate # TODO: remove after charmcraft 3.3 stable release path-to-charm-directory: ${{ matrix.charm }}