feat: Add bundle url as input and use latest as default (#158)

---------
Co-authored-by: Orfeas Kourkakis <[email protected]>

README.md

@@ -22,6 +22,7 @@ found in the [Run the tests](#run-the-tests) section.
       * [Using a remote commit](#run-tests-from-a-remote-commit)
       * [Using a local copy](#run-tests-from-local-copy)
       * [A subset of UATs](#run-a-subset-of-uats)
+      * [Specify a different bundle](#specify-a-different-bundle)
       * [Kubeflow UATs](#run-kubeflow-uats)
       * [MLflow UATs](#run-mlflow-uats)
    * [NVIDIA GPU UAT](#nvidia-gpu-uat)
@@ -65,9 +66,17 @@ As mentioned before, when it comes to running the tests, you've got 2 options:
 * Running the tests on an existing cluster using the `driver` along with the provided automation
 
 NOTE: Depending on the version of Charmed Kubeflow you want to test, make sure to checkout to the appropriate branch with `git checkout`:
+- Charmed Kubeflow 1.9 -> `track/1.9`
 - Charmed Kubeflow 1.8 -> `track/1.8`
 - Charmed Kubeflow 1.7 -> `track/1.7`
 
+`main` branch is generally used for testing against the `latest/edge` track of the bundle.   
+
+As part of the tests, the UATs checks that the version of the applications are the ones expected for the various tracks. The different branches
+above point to a different bundle from the [bundle-kubeflow](https://github.com/canonical/bundle-kubeflow) repository to compare the 
+channels in the deployment being tested. `main` branch also provides ability to specify the of the bundle to be used for checking by providing
+the `--bundle` argument for the tox entrypoints.
+
 ### Running inside a Notebook
 
 * Create a new Notebook using the `jupyter-scipy` image:
@@ -148,6 +157,19 @@ This simulates the behaviour of running `pytest -k "some filter"` directly on th
 You can read more about the options provided by Pytest in the corresponding section of the
 [documentation](https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags).
 
+#### Specify a different bundle
+
+To provide a different bundle to be used to check that the deployment has the correct channel version, 
+use the `--bundle` flag, e.g.
+
+```bash
+tox -e uats-remote -- --bundle <my-bundle>
+```
+
+The `<my-bundle>` can be replaced by either a URL, e.g. `http://...`, or a local file, `file:/path/to/file`. Note that the local file path must be accessible when running the tests. 
+
+This flag is currently only provided on main branch and tracks 1.9+. 
+
 #### Run Kubeflow UATs
 
 In order to only run the Kubeflow-specific tests (i.e. no MLflow integration) you can use the

driver/conftest.py

@@ -3,6 +3,8 @@
 
 from _pytest.config.argparsing import Parser
 
+BUNDLE_URL = "https://raw.githubusercontent.com/canonical/bundle-kubeflow/refs/heads/main/releases/latest/edge/bundle.yaml"
+
 
 def pytest_addoption(parser: Parser):
     """Add pytest options.
@@ -58,3 +60,8 @@ def pytest_addoption(parser: Parser):
         default="kubeflow",
         help="Provide the name of the namespace/juju model where kubeflow is deployed.",
     )
+    parser.addoption(
+        "--bundle",
+        default=BUNDLE_URL,
+        help="Provide the bundle to be used during the check. You can use a URL, e.g. http://..., or a local file, file:/path/to/file. If empty, the check is skipped",
+    )

driver/test_kubeflow_workloads.py

@@ -62,15 +62,27 @@
 
 KFP_PODDEFAULT_NAME = "access-ml-pipeline"
 
-BUNDLE_URL = "https://raw.githubusercontent.com/canonical/bundle-kubeflow/refs/heads/main/releases/1.9/stable/bundle.yaml"
-
 
 @pytest.fixture(scope="module")
-def charm_list():
-    if not (response := requests.get(BUNDLE_URL)) or (response.status_code != 200):
+def charm_list(request):
+    url = request.config.getoption("--bundle")
+
+    if not url:
         return {}
 
-    bundle = yaml.safe_load(response.content.decode("utf-8"))
+    if url.startswith("http"):
+        if not (response := requests.get(url)) or (response.status_code != 200):
+            logging.warning(f"Bundle file {url} could not be downloaded")
+            return {}
+
+        bundle = yaml.safe_load(response.content.decode("utf-8"))
+    else:
+        if not (filename := re.compile("^file:").sub("", url)) or not Path(filename).exists():
+            logging.warning(f"Bundle file {filename} does not exist")
+            return {}
+
+        with open(filename, "r") as fid:
+            bundle = yaml.safe_load(fid)
 
     return {
         app_name: charm["channel"].split("/")[0] + "/*"
@@ -174,13 +186,26 @@ def create_poddefault_on_toleration(request, lightkube_client):
 
 @pytest.mark.abort_on_fail
 async def test_bundle_correctness(ops_test, kubeflow_model, charm_list):
+    """Test that the correct bundle is selected.
+
+    Tests are specific to each Charmed Kubeflow version release. This test makes sure that
+    the correct version of the bundle, consistent with the tests, is specified. In order to
+    check the correctness, we use a YAML bundle that is pulled from the correct URL in the
+    bundle-kubeflow repository. This value can be overridden using the `--bundle` argument.
+    """
+
+    if not charm_list:
+        pytest.skip("charm_list empty. Cannot test bundle correctness")
 
     model = await ops_test.track_model("kubeflow", model_name=kubeflow_model, use_existing=True)
     status = await model.get_status()
 
     # Check that the version is the one expected by this set of tests
     for name, channel_regex in charm_list.items():
-        assert re.compile(channel_regex).match(status["applications"][name]["charm-channel"])
+        app_channel = status["applications"][name]["charm-channel"]
+        assert re.compile(channel_regex).match(
+            app_channel
+        ), f"Failed bundle correctness check. Expected: {channel_regex} Found: {app_channel}"
 
     # Check that everything is active/idle
     await ops_test.model.wait_for_idle(