RHOAIENG-18848: chore(tests/containers): initial kubernetes/openshift deployment support

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -296,10 +296,10 @@ jobs:
       - name: Install deps
         run: poetry install --sync
 
-      - name: Run container tests (in PyTest)
+      - name: Run Testcontainers container tests (in PyTest)
         run: |
           set -Eeuxo pipefail
-          poetry run pytest --capture=fd tests/containers --image="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
+          poetry run pytest --capture=fd tests/containers -m 'not openshift' --image="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
         env:
           DOCKER_HOST: "unix:///var/run/podman/podman.sock"
           TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/podman/podman.sock"
@@ -445,6 +445,16 @@ jobs:
           kubectl wait deployments --all --all-namespaces --for=condition=Available --timeout=100s
           kubectl wait pods --all --all-namespaces --for=condition=Ready --timeout=100s
 
+      - name: "Install local-path provisioner"
+        if: ${{ steps.have-tests.outputs.tests == 'true' }}
+        run: |
+          set -Eeuxo pipefail
+          kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.31/deploy/local-path-storage.yaml
+          kubectl wait deployments --all --namespace=local-path-storage --for=condition=Available --timeout=100s
+          # https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/
+          kubectl get storageclass
+          kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+
       - name: "Run image tests"
         if: ${{ steps.have-tests.outputs.tests == 'true' }}
         run: python3 ci/cached-builds/make_test.py --target ${{ inputs.target }}
@@ -455,6 +465,18 @@ jobs:
 
       # endregion
 
+      - name: Run OpenShift container tests (in PyTest)
+        if: ${{ steps.have-tests.outputs.tests == 'true' }}
+        run: |
+          set -Eeuxo pipefail
+          poetry run pytest --capture=fd tests/containers -m 'openshift' --image="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
+        env:
+          # TODO(jdanek): this Testcontainers stuff should not be necessary but currently it has to be there
+          DOCKER_HOST: "unix:///var/run/podman/podman.sock"
+          TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/podman/podman.sock"
+          # pulling the Ryuk container from docker.io introduces CI flakiness
+          TESTCONTAINERS_RYUK_DISABLED: "true"
+
       # region Trivy vulnerability scan
 
       - name: Run Trivy vulnerability scanner

README.md

@@ -105,15 +105,15 @@ sudo dnf install podman
 systemctl --user start podman.service
 systemctl --user status podman.service
 systemctl --user status podman.socket
-DOCKER_HOST=unix:///run/user/$UID/podman/podman.sock poetry run pytest tests/containers --image quay.io/opendatahub/workbench-images@sha256:e98d19df346e7abb1fa3053f6d41f0d1fa9bab39e49b4cb90b510ca33452c2e4
+DOCKER_HOST=unix:///run/user/$UID/podman/podman.sock poetry run pytest tests/containers -m 'not openshift' --image quay.io/opendatahub/workbench-images@sha256:e98d19df346e7abb1fa3053f6d41f0d1fa9bab39e49b4cb90b510ca33452c2e4
 
 # Mac OS
 brew install podman
 podman machine init
 podman machine set --rootful=false
 sudo podman-mac-helper install
 podman machine start
-poetry run pytest tests/containers --image quay.io/opendatahub/workbench-images@sha256:e98d19df346e7abb1fa3053f6d41f0d1fa9bab39e49b4cb90b510ca33452c2e4
+poetry run pytest tests/containers -m 'not openshift' --image quay.io/opendatahub/workbench-images@sha256:e98d19df346e7abb1fa3053f6d41f0d1fa9bab39e49b4cb90b510ca33452c2e4
 ```
 
 When using lima on macOS, it might be useful to give yourself access to rootful podman socket

pyproject.toml

@@ -8,6 +8,7 @@ package-mode = false
 
 [tool.poetry.dependencies]
 python = "~3.12"
+requests = "^2.32.3"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -18,6 +19,8 @@ pyfakefs = "^5.7.4"
 testcontainers = "^4.9.1"
 docker = "^7.1.0"
 podman = "^5.2.0"
+kubernetes = "^31.0.0"
+openshift-python-wrapper = "^11.0.19"
 pydantic = "^2.10.6"
 requests = "^2.32.3"
 

pytest.ini

@@ -15,3 +15,5 @@ log_cli_level = INFO
 
 log_file = logs/pytest-logs.txt
 log_file_level = DEBUG
+
+markers = openshift

tests/containers/base_image_test.py

@@ -11,12 +11,13 @@
 import textwrap
 from typing import TYPE_CHECKING, Any, Callable
 
-import pytest
 import testcontainers.core.container
 import testcontainers.core.waiting_utils
 
 from tests.containers import docker_utils
 
+import pytest
+
 logging.basicConfig(level=logging.DEBUG)
 LOGGER = logging.getLogger(__name__)
 
@@ -72,7 +73,8 @@ def check_elf_file():
                         if "not found" in line:
                             unsatisfied_deps.append((dlib, line.strip()))
                     assert output
-                print("OUTPUT>", json.dumps({"dir": path, "count_scanned": count_scanned, "unsatisfied": unsatisfied_deps}))
+                print("OUTPUT>",
+                      json.dumps({"dir": path, "count_scanned": count_scanned, "unsatisfied": unsatisfied_deps}))
 
         try:
             container.start()
@@ -117,6 +119,7 @@ def test_oc_command_runs(self, image: str):
         logging.debug(output.decode())
         assert ecode == 0
 
+    # @pytest.mark.environmentss("docker")
     def test_oc_command_runs_fake_fips(self, image: str, subtests: pytest_subtests.SubTests):
         """Establishes a best-effort fake FIPS environment and attempts to execute `oc` binary in it.
 
@@ -140,7 +143,8 @@ def test_oc_command_runs_fake_fips(self, image: str, subtests: pytest_subtests.S
             # if /proc/sys/crypto/fips_enabled exists, only replace this file,
             # otherwise (Ubuntu case), assume entire /proc/sys/crypto does not exist
             if platform.system().lower() == "darwin" or pathlib.Path("/proc/sys/crypto/fips_enabled").exists():
-                container.with_volume_mapping(str(tmp_crypto / 'crypto' / 'fips_enabled'), "/proc/sys/crypto/fips_enabled", mode="ro,z")
+                container.with_volume_mapping(str(tmp_crypto / 'crypto' / 'fips_enabled'),
+                                              "/proc/sys/crypto/fips_enabled", mode="ro,z")
             else:
                 container.with_volume_mapping(str(tmp_crypto), "/proc/sys", mode="ro,z")
 

tests/containers/cancellation_token.py

@@ -0,0 +1,37 @@
+import os
+import threading
+
+
+class CancellationToken:
+    """Flag to signal a thread it should cancel itself.
+    This cooperative cancellation pattern is commonly used in c# and go
+    See https://learn.microsoft.com/en-us/dotnet/api/system.threading.cancellationtoken?view=net-9.0
+    """
+
+    def __init__(self):
+        # consider using the wrapt.synchronized decorator
+        # https://github.com/GrahamDumpleton/wrapt/blob/develop/blog/07-the-missing-synchronized-decorator.md
+        self._lock = threading.Lock()
+        self._canceled = False
+        # something selectable avoids having to use short timeout in select
+        self._read_fd, self._write_fd = os.pipe()
+
+    def fileno(self):
+        """This lets us use the token in select() calls"""
+        return self._read_fd
+
+    @property
+    def cancelled(self):
+        with self._lock:
+            return self._canceled
+
+    def cancel(self):
+        with self._lock:
+            os.write(self._write_fd, b'x')
+            self._canceled = True
+
+    def __del__(self):
+        # consider https://docs.python.org/3/library/weakref.html#weakref.finalize
+        with self._lock:
+            os.close(self._read_fd)
+            os.close(self._write_fd)