[DPE-4530] CI testing both forward (Juju) and backwards (focal) (#172)

* Increase pipeline coverage

* Test charms handling multiple ubuntu versions

* Documenting multi-versions test executions

* Disabling build as we're not using the cache

* Changes on PR request

* Adding missing unittests

* More changes on PR request

.github/workflows/ci.yaml

@@ -47,65 +47,74 @@ jobs:
         env:
           LIBJUJU_VERSION_SPECIFIER: "==${{ matrix.juju-version.libjuju-version }}"
 
-  build:
-    name: Build charms
-    needs:
-      - lint
-      - unit-test
-    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2
-
   integration-test:
     strategy:
       fail-fast: false
       matrix:
+        ubuntu-versions:
+          # Update whenever charmcraft.yaml is changed
+          - series: jammy
+            bases-index: 0
+          - series: focal
+            bases-index: 1
         tox-environments:
-          - integration-database
+          - integration-db
+          - integration-upgrade-1
+          - integration-upgrade-2
+          - integration-upgrade-3
+          - integration-upgrade-4
+          # To be enabled after data_interfaces v040
+          # - integration-upgrade-5
+          # - integration-upgrade-6
+          # - integration-upgrade-7
+          - integration-upgrade-databag
           - integration-s3
           - integration-opensearch
           - integration-kafka
         juju-version:
           - juju-bootstrap-option: "2.9.44"
             juju-snap-channel: "2.9/stable"
-            libjuju-version: "2.9.42.4"
-          - juju-bootstrap-option: "3.1.6"
+            libjuju-version: "2.9.49.0"
+          - juju-bootstrap-option: "3.1.8"
             juju-snap-channel: "3.1/stable"
             libjuju-version: "3.2.2"
-        include:
-          - {tox-environments: integration-interfaces-upgrade-from-version-1-earlier,
-              juju-version:
-              {juju-snap-channel: "3.1/stable",
-                juju-bootstrap-option: "3.1.7",
-                libjuju-version: "3.2.2"}}
-          - {tox-environments: integration-interfaces-upgrade-from-version-2-earlier,
-              juju-version:
-              {juju-snap-channel: "3.1/stable",
-                juju-bootstrap-option: "3.1.7",
-                libjuju-version: "3.2.2"}}
-          - {tox-environments: integration-interfaces-upgrade-from-version-3-earlier,
-              juju-version:
-              {juju-snap-channel: "3.1/stable",
-                juju-bootstrap-option: "3.1.7",
-                libjuju-version: "3.2.2"}}
-          - {tox-environments: integration-interfaces-upgrade-from-version-4-earlier,
-              juju-version:
-              {juju-snap-channel: "3.1/stable",
-                juju-bootstrap-option: "3.1.7",
-                libjuju-version: "3.2.2"}}
-          - {tox-environments: integration-upgrades-databag-to-secrets,
-              juju-version:
-              {juju-snap-channel: "3.1/stable",
-                juju-bootstrap-option: "3.1.7",
-                libjuju-version: "3.2.2"}}
-          - {tox-environments: integration-secrets,
-              juju-version:
-              {juju-snap-channel: "3.1/stable",
-                juju-bootstrap-option: "3.1.6",
-                libjuju-version: "3.2.2"}}
-    name: ${{ matrix.tox-environments }} Juju ${{ matrix.juju-version.juju-snap-channel}} -- libjuju ${{ matrix.juju-version.libjuju-version }}
+          - juju-bootstrap-option: "3.2.4"
+            juju-snap-channel: "3.2/stable"
+            libjuju-version: "3.2.3.0"
+          - juju-bootstrap-option: "3.3.5"
+            juju-snap-channel: "3.3/stable"
+            libjuju-version: "3.3.1.1"
+          - juju-bootstrap-option: "3.4.3"
+            juju-snap-channel: "3.4/stable"
+            libjuju-version: "3.4.0.0"
+          - juju-bootstrap-option: "3.5.1"
+            juju-snap-channel: "3.5/stable"
+            libjuju-version: "3.5.0.0"
+        exclude:
+          - tox-environments: integration-upgrade-1
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-2
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-3
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-4
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-5
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-6
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-7
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-upgrade-databag
+            juju-version: {juju-snap-channel: "2.9/stable"}
+          - tox-environments: integration-opensearch
+            ubuntu-versions: {series: focal}
+          - tox-environments: integration-kafka
+            ubuntu-versions: {series: focal}
+    name: ${{ matrix.tox-environments }} Juju ${{ matrix.juju-version.juju-snap-channel}} -- ${{ matrix.ubuntu-versions.series }}
     needs:
       - lint
       - unit-test
-      - build
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
@@ -136,7 +145,7 @@ jobs:
           fi
       - name: Run integration tests
         # set a predictable model name so it can be consumed by charm-logdump-action
-        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing
+        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing  --os-series=${{ matrix.ubuntu-versions.series }} --build-bases-index=${{ matrix.ubuntu-versions.bases-index }}
         env:
           CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
           LIBJUJU_VERSION_SPECIFIER: "==${{ matrix.juju-version.libjuju-version }}"

CONTRIBUTING.md

@@ -18,7 +18,7 @@ this operator.
   - code quality
   - test coverage
   - user experience for Juju administrators this charm.
-- Please help us out in ensuring easy to review branches by rebasing your pull
+- Please help us out in ensuring easy to review branches by re-basing your pull
   request branch onto the `main` branch. This also avoids merge commits and
   creates a linear Git commit history.
 
@@ -37,10 +37,32 @@ source venv/bin/activate
 tox run -e format        # update your code according to linting rules
 tox run -e lint          # code style
 tox run -e unit          # unit tests
-tox run -e integration   # integration tests
+tox run -e integration-*   # integration tests
 tox                      # runs 'lint' and 'unit' environments
 ```
 
+### Adding new tests
+
+In case your tests are re-using existing test charms with no modifications, feel free to ignore this section.
+
+For test charms that may support multiple Ubuntu versions (typically for libraries that are expected to work across
+legacy versions) the following mechanism is available.
+
+By default, with no parameters the first Ubuntu version is taken, that's specified in the helper charm's `charmcraft.yaml`.
+The current default is `jammy` with `base_index: 0` (i.e. being specified as the first `build-on/run-on` environment).
+Ordering goes in a way that newest version comes first, older ones decreasingly after.
+(This allows for meaningful defaults for helper charms for libs that only support newer/latest Ubuntu series.)
+
+In case any further Ubuntu versions are to be used when executing the tests, the following `pytest` parameters are to be added
+at execution time
+ - `os_series`: The name of the Ubuntu series to be used for build/deploy (default: `jammy`)
+ - `build_bases_index`: The number of item in order (counting from `0`) referring to the `db_libs_series` specified in the
+    helper charm's `charmcraft.yaml`
+
+NOTE: In case using the mechanism above, make sure that the `os_series` fixture's value is passed for the
+`series` option to your charm when deploying it in the test pipeline execution (typically: the `build_and_deploy` test function).
+
+
 ## Build charm
 
 Build the charm in this git repository using:

charmcraft.yaml

@@ -11,8 +11,8 @@ parts:
       - cargo
 bases:
   - build-on:
-    - name: "ubuntu"
-      channel: "22.04"
+      - name: "ubuntu"
+        channel: "22.04"
     run-on:
-    - name: "ubuntu"
-      channel: "22.04"
+      - name: "ubuntu"
+        channel: "22.04"

tests/conftest.py

@@ -1,5 +1,6 @@
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
+import argparse
 import os
 from importlib.metadata import version
 from unittest.mock import PropertyMock
@@ -9,6 +10,32 @@
 from pytest_mock import MockerFixture
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--os-series", help="Ubuntu series for dp libs charm (e.g. jammy)", default="jammy"
+    )
+    parser.addoption(
+        "--build-bases-index",
+        type=int,
+        help="Index of charmcraft.yaml base that matches --os-series",
+        default=0,
+    )
+
+
+def pytest_configure(config):
+    if (config.option.os_series is None) ^ (config.option.build_bases_index is None):
+        raise argparse.ArgumentError(
+            None,
+            "--os-series and --build-bases-index must be given together",
+        )
+    # Note: Update defaults whenever charmcraft.yaml is changed
+    valid_combinations = [(0, "jammy"), (1, "focal")]
+    if (config.option.build_bases_index, config.option.os_series) not in valid_combinations:
+        raise argparse.ArgumentError(
+            None, f"Only base index combinations {valid_combinations} are accepted."
+        )
+
+
 @pytest.fixture(autouse=True)
 def juju_has_secrets(mocker: MockerFixture):
     """This fixture will force the usage of secrets whenever run on Juju 3.x.

tests/integration/application-charm/charmcraft.yaml

@@ -3,12 +3,21 @@
 
 type: charm
 bases:
+  # Whenever "bases" is changed:
+  # - Update tests/integration/conftest.py::pytest_configure()
+  # - Update .github/workflow/ci.yaml integration-test matrix
   - build-on:
       - name: "ubuntu"
         channel: "22.04"
     run-on:
       - name: "ubuntu"
         channel: "22.04"
+  - build-on:
+      - name: "ubuntu"
+        channel: "20.04"
+    run-on:
+      - name: "ubuntu"
+        channel: "20.04"
 parts:
   charm:
     charm-binary-python-packages:

tests/integration/application-s3-charm/charmcraft.yaml

@@ -3,9 +3,18 @@
 
 type: charm
 bases:
+  # Whenever "bases" is changed:
+  # - Update tests/integration/conftest.py::pytest_configure()
+  # - Update .github/workflow/ci.yaml integration-test matrix
   - build-on:
       - name: "ubuntu"
         channel: "22.04"
     run-on:
       - name: "ubuntu"
         channel: "22.04"
+  - build-on:
+      - name: "ubuntu"
+        channel: "20.04"
+    run-on:
+      - name: "ubuntu"
+        channel: "20.04"

tests/integration/conftest.py

@@ -2,7 +2,7 @@
 # Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-import json
+import logging
 import os
 import shutil
 from datetime import datetime
@@ -12,22 +12,38 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="session")
+def dp_libs_ubuntu_series(pytestconfig) -> str:
+    if pytestconfig.option.os_series:
+        return pytestconfig.option.os_series
+
 
 @pytest.fixture(scope="module")
-def ops_test(ops_test: OpsTest) -> OpsTest:
-    if os.environ.get("CI") == "true":
-        # Running in GitHub Actions; skip build step
-        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
-        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
-
-        async def build_charm(charm_path, bases_index: int = None) -> Path:
-            for charm in packed_charms:
-                if Path(charm_path) == Path(charm["directory_path"]):
-                    if bases_index is None or bases_index == charm["bases_index"]:
-                        return charm["file_path"]
-            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
-
-        ops_test.build_charm = build_charm
+def ops_test(ops_test: OpsTest, pytestconfig) -> OpsTest:
+    """Re-defining OpsTest.build_charm in a way that it takes CI caching and build parameters into account.
+
+    Build parameters (for charms available for multiple OS versions) are considered both when building the
+    charm, or when fetching pre-built, CI cached version of it.
+    """
+    _build_charm = ops_test.build_charm
+
+    # Add bases_index option (indicating which OS version to use)
+    # when building the charm within the scope of the test run
+    async def build_charm(charm_path, bases_index: int = None) -> Path:
+        if not bases_index and pytestconfig.option.build_bases_index:
+            bases_index = pytestconfig.option.build_bases_index
+
+        logger.info(f"Building charm {charm_path} with base index {bases_index}")
+
+        return await _build_charm(
+            charm_path,
+            bases_index=pytestconfig.option.build_bases_index,
+        )
+
+    ops_test.build_charm = build_charm
     return ops_test
 
 
@@ -169,7 +185,7 @@ def fetch_old_versions():
     check_call("git clone https://github.com/canonical/data-platform-libs.git", shell=True)
     os.chdir("data-platform-libs")
     last_commits = check_output(
-        "git show --pretty=format:'%h' --no-patch -15", shell=True, universal_newlines=True
+        "git show --pretty=format:'%h' --no-patch -25", shell=True, universal_newlines=True
     ).split()
 
     versions = []
@@ -185,7 +201,7 @@ def fetch_old_versions():
             shutil.copyfile(src_path, f"{data_path}.v{version}")
             versions.append(version)
 
-        if len(versions) == 4:
+        if len(versions) == 7:
             break
 
     os.chdir(cwd)

tests/integration/database-charm/charmcraft.yaml

@@ -3,12 +3,21 @@
 
 type: charm
 bases:
+  # Whenever "bases" is changed:
+  # - Update tests/integration/conftest.py::pytest_configure()
+  # - Update .github/workflow/ci.yaml integration-test matrix
   - build-on:
       - name: "ubuntu"
         channel: "22.04"
     run-on:
       - name: "ubuntu"
         channel: "22.04"
+  - build-on:
+      - name: "ubuntu"
+        channel: "20.04"
+    run-on:
+      - name: "ubuntu"
+        channel: "20.04"
 parts:
   charm:
     charm-binary-python-packages: [psycopg2-binary==2.9.3]

tests/integration/database-charm/src/charm.py

@@ -13,6 +13,7 @@
 import string
 from random import randrange
 from time import sleep
+from typing import Optional
 
 import psycopg2
 from ops import Relation
@@ -119,7 +120,7 @@ def __init__(self, *args):
         )
 
     @property
-    def peer_relation(self) -> Relation | None:
+    def peer_relation(self) -> Optional[Relation]:
         """The cluster peer relation."""
         return self.model.get_relation(PEER)