draft: PoC for refactor CI epic

.github/workflows/integrate.yaml

@@ -66,20 +66,42 @@ jobs:
       - run: python3 -m pip install tox
       - run: tox -e ${{ matrix.charm }}-unit
 
+  build:
+    name: Build charm
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    strategy:
+      fail-fast: false
+      matrix:
+        charm:
+          - kfp-api
+          - kfp-metadata-writer
+          - kfp-persistence
+          - kfp-profile-controller
+          - kfp-schedwf
+          - kfp-ui
+          - kfp-viewer
+          - kfp-viz
+    with:
+      cache: ${{ github.event_name == 'pull_request' }}
+      charmcraft-snap-channel: 3.x/edge
+      path-to-charm-directory: ./charms/${{ matrix.charm }}
+
+
   integration:
     name: Integration tests (microk8s)
     runs-on: ubuntu-20.04
+    needs: [build]
     strategy:
       fail-fast: false
       matrix:
         charm:
+          - kfp-api
+          - kfp-metadata-writer
           - kfp-persistence
           - kfp-profile-controller
-          - kfp-api
           - kfp-schedwf
-          - kfp-viewer
           - kfp-ui
-          - kfp-metadata-writer
+          - kfp-viewer
           - kfp-viz
     steps:
       # Ideally we'd use self-hosted runners, but this effort is still not stable
@@ -90,7 +112,7 @@ jobs:
       - name: Maximise GH runner space
         uses: jlumbroso/[email protected]
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
@@ -101,12 +123,20 @@ jobs:
           # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845
           charmcraft-channel: 3.x/stable
 
+      - name: Download packed charm(s)
+        id: download-charms
+        timeout-minutes: 5
+        uses: actions/download-artifact@v4
+        with:
+          pattern: packed-charm-cache-true-.-charms-${{ matrix.charm }}-*
+          merge-multiple: true
+
       - name: Integration tests
         run: |
           # Requires the model to be called kubeflow due to
           # https://github.com/canonical/kfp-operators/issues/389
           juju add-model kubeflow
-          sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow"
+          sg snap_microk8s -c "tox -vve ${{ matrix.charm }}-integration-using-packed-charms -- --model kubeflow"
 
       - name: Collect charm debug artifacts
         uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main
@@ -144,11 +174,19 @@ jobs:
           charmcraft-channel: 3.x/stable
           microk8s-addons: "dns hostpath-storage rbac metallb:10.64.140.43-10.64.140.49"
 
+      - name: Download packed charm(s)
+        id: download-charms
+        timeout-minutes: 5
+        uses: actions/download-artifact@v4
+        with:
+          pattern: packed-charm-cache-true-.-*
+          merge-multiple: true
+
       - name: Run test
         run: |
           # Requires the model to be called kubeflow due to kfp-viewer
           juju add-model kubeflow
-          sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }} -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2"
+          sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }}-using-packed-charms -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2"
 
       - name: Get all
         run: kubectl get all -A

.github/workflows/publish.yaml

@@ -41,14 +41,12 @@ jobs:
         run: bash .github/workflows/get-charm-paths.sh
 
 
-  publish-charm:
-    name: Publish Charm
+  define-channel:
+    name: Define destination channel
     runs-on: ubuntu-20.04
     needs: get-charm-paths
-    strategy:
-      fail-fast: false
-      matrix:
-        charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }}
+    outputs:
+      destination-channel: ${{ steps.parse-inputs.outputs.destination_channel }}
 
     steps:
       - name: Checkout
@@ -72,24 +70,19 @@ jobs:
           echo "setting output of destination_channel=$destination_channel"
           echo "::set-output name=destination_channel::$destination_channel"
 
-          # tag_prefix
-          # if charm_path = ./ --> tag_prefix = '' (null)
-          # if charm_path != ./some-charm (eg: a charm in a ./charms dir) --> tag_prefix = 'some-charm'
-          if [ ${{ matrix.charm-path }} == './' ]; then
-            tag_prefix=''
-          else
-            tag_prefix=$(basename ${{ matrix.charm-path }} )
-          fi
-          echo "setting output of tag_prefix=$tag_prefix"
-          echo "::set-output name=tag_prefix::$tag_prefix"
-
-      - name: Upload charm to charmhub
-        uses: canonical/charming-actions/[email protected]
-        with:
-          credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          charm-path: ${{ matrix.charm-path }}
-          channel: ${{ steps.parse-inputs.outputs.destination_channel }}
-          tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }}
-          # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845
-          charmcraft-channel: 3.x/stable
+  release:
+    name: Release charm
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    needs: [define-channel, get-charm-paths]
+    strategy:
+      fail-fast: false
+      matrix:
+        charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }}
+    with:
+      channel: ${{ needs.define-channel.outputs.destination-channel }}
+      artifact-prefix: packed-charm-cache-true-.-charms-${{ matrix.charm-path }}
+      create-github-release: ${{ github.event_name == 'push' }}
+    secrets:
+      charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+    permissions:
+      contents: write  # Needed to create GitHub release

charms/kfp-api/tests/integration/test_charm.py

@@ -2,6 +2,7 @@
 # See LICENSE file for licensing details.
 
 import logging
+import os
 from pathlib import Path
 
 import pytest
@@ -15,6 +16,7 @@
     get_alert_rules,
 )
 from pytest_operator.plugin import OpsTest
+from utils import get_packed_charms
 
 logger = logging.getLogger(__name__)
 
@@ -39,14 +41,24 @@
 MYSQL_TRUST = True
 
 
+@pytest.fixture
+def use_packed_charms() -> str:
+    """Return environment variable `USE_PACKED_CHARMS`. If it's not found, return `false`."""
+    return os.environ.get("USE_PACKED_CHARMS", "false").replace('"', "")
+
+
 class TestCharm:
     """Integration test charm"""
 
     @pytest.mark.abort_on_fail
-    async def test_build_and_deploy(self, ops_test: OpsTest):
+    async def test_build_and_deploy(self, ops_test: OpsTest, use_packed_charms):
         """Deploy kfp-api with required charms and relations."""
-        built_charm_path = await ops_test.build_charm("./")
-        logger.info(f"Built charm {built_charm_path}")
+        charm_path = "."
+        if use_packed_charms.lower() == "true":
+            built_charm_path = await get_packed_charms(charm_path)
+        else:
+            built_charm_path = await ops_test.build_charm(charm_path)
+            logger.info(f"Built charm {built_charm_path}")
 
         image_path = METADATA["resources"]["oci-image"]["upstream-source"]
         resources = {"oci-image": image_path}

charms/kfp-api/tests/integration/utils.py

@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Utils functions for integration tests. Should be moved to chisme."""
+
+import os
+import pathlib
+import subprocess
+import typing
+
+
+async def get_packed_charms(
+    charm_path: typing.Union[str, os.PathLike], bases_index: int = None
+) -> pathlib.Path:
+    """Simplified version of https://github.com/canonical/data-platform-workflows/blob/06f252ea079edfd055cee236ede28c237467f9b0/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py#L22."""  # noqa: E501
+    charm_path = pathlib.Path(charm_path)
+    # namespace-node-affinity_ubuntu-20.04-amd64.charm
+    # <metadata-name>_<base>-<architecture>.charm
+    architecture = subprocess.run(
+        ["dpkg", "--print-architecture"],
+        capture_output=True,
+        check=True,
+        encoding="utf-8",
+    ).stdout.strip()
+    assert architecture in ("amd64", "arm64")
+    packed_charms = list(charm_path.glob(f"*-{architecture}.charm"))
+    if len(packed_charms) == 1:
+        # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms
+        # to begin with `./` or `/` to distinguish them from Charmhub charms.
+        # Therefore, we need to return an absolute path—a relative `pathlib.Path` does not start
+        # with `./` when cast to a str.
+        # (python-libjuju model.deploy() expects a str but will cast any input to a str as a
+        # workaround for pytest-operator's non-compliant `build_charm` return type of
+        # `pathlib.Path`.)
+        return packed_charms[0].resolve(strict=True)
+    elif len(packed_charms) > 1:
+        message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}."  # noqa: E501
+        if bases_index is None:
+            message += " Specify `bases_index`"
+        raise ValueError(message)
+    else:
+        raise ValueError(
+            f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}"
+        )

charms/kfp-api/tox.ini

@@ -7,7 +7,7 @@ max-line-length = 100
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-envlist = fmt, lint, unit, integration
+envlist = fmt, lint, unit, integration{-using-packed-charms,}
 
 [vars]
 all_path = {[vars]src_path} {[vars]tst_path}
@@ -73,8 +73,10 @@ deps =
     -r requirements-unit.txt
 description = Run unit tests
 
-[testenv:integration]
+[testenv:integration{-using-packed-charms,}]
 commands = pytest -vv --tb native --asyncio-mode=auto {[vars]tst_path}integration --log-cli-level=INFO -s {posargs}
+setenv = 
+    using-packed-charms: USE_PACKED_CHARMS = "true"
 deps =
     -r requirements-integration.txt
 description = Run integration tests

charms/kfp-metadata-writer/tests/integration/test_charm.py

@@ -2,6 +2,7 @@
 # See LICENSE file for licensing details.
 
 import logging
+import os
 from pathlib import Path
 
 import pytest
@@ -12,6 +13,7 @@
     deploy_and_assert_grafana_agent,
 )
 from pytest_operator.plugin import OpsTest
+from utils import get_packed_charms
 
 METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
 CHARM_ROOT = "."
@@ -25,10 +27,20 @@
 log = logging.getLogger(__name__)
 
 
+@pytest.fixture
+def use_packed_charms() -> str:
+    """Return environment variable `USE_PACKED_CHARMS`. If it's not found, return `false`."""
+    return os.environ.get("USE_PACKED_CHARMS", "false").replace('"', "")
+
+
 @pytest.mark.abort_on_fail
-async def test_build_and_deploy_with_relations(ops_test: OpsTest):
-    built_charm_path = await ops_test.build_charm(CHARM_ROOT)
-    log.info(f"Built charm {built_charm_path}")
+async def test_build_and_deploy_with_relations(ops_test: OpsTest, use_packed_charms):
+    charm_path = "."
+    if use_packed_charms.lower() == "true":
+        built_charm_path = await get_packed_charms(charm_path)
+    else:
+        built_charm_path = await ops_test.build_charm(charm_path)
+        log.info(f"Built charm {built_charm_path}")
 
     image_path = METADATA["resources"]["oci-image"]["upstream-source"]
     resources = {"oci-image": image_path}

charms/kfp-metadata-writer/tests/integration/utils.py

@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Utils functions for integration tests. Should be moved to chisme."""
+
+import os
+import pathlib
+import subprocess
+import typing
+
+
+async def get_packed_charms(
+    charm_path: typing.Union[str, os.PathLike], bases_index: int = None
+) -> pathlib.Path:
+    """Simplified version of https://github.com/canonical/data-platform-workflows/blob/06f252ea079edfd055cee236ede28c237467f9b0/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py#L22."""  # noqa: E501
+    charm_path = pathlib.Path(charm_path)
+    # namespace-node-affinity_ubuntu-20.04-amd64.charm
+    # <metadata-name>_<base>-<architecture>.charm
+    architecture = subprocess.run(
+        ["dpkg", "--print-architecture"],
+        capture_output=True,
+        check=True,
+        encoding="utf-8",
+    ).stdout.strip()
+    assert architecture in ("amd64", "arm64")
+    packed_charms = list(charm_path.glob(f"*-{architecture}.charm"))
+    if len(packed_charms) == 1:
+        # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms
+        # to begin with `./` or `/` to distinguish them from Charmhub charms.
+        # Therefore, we need to return an absolute path—a relative `pathlib.Path` does not start
+        # with `./` when cast to a str.
+        # (python-libjuju model.deploy() expects a str but will cast any input to a str as a
+        # workaround for pytest-operator's non-compliant `build_charm` return type of
+        # `pathlib.Path`.)
+        return packed_charms[0].resolve(strict=True)
+    elif len(packed_charms) > 1:
+        message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}."  # noqa: E501
+        if bases_index is None:
+            message += " Specify `bases_index`"
+        raise ValueError(message)
+    else:
+        raise ValueError(
+            f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}"
+        )

charms/kfp-metadata-writer/tox.ini

@@ -7,7 +7,7 @@ max-line-length = 100
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-envlist = fmt, lint, unit, integration
+envlist = fmt, lint, unit, integration{-using-packed-charms,}
 
 [vars]
 all_path = {[vars]src_path} {[vars]tst_path}
@@ -73,8 +73,10 @@ deps =
     -r requirements-unit.txt
 description = Run unit tests
 
-[testenv:integration]
+[testenv:integration{-using-packed-charms,}]
 commands = pytest -v --tb native --asyncio-mode=auto {[vars]tst_path}integration --log-cli-level=INFO -s {posargs}
+setenv = 
+    using-packed-charms: USE_PACKED_CHARMS = "true"
 deps =
     -r requirements-integration.txt
 description = Run integration tests