Merge pull request #1002 from Aiven-Open/nosahama/EC-666/docker-tests…

…-multi-python-version

ci: multi-python version docker image for GHA matrix

.dockerignore

@@ -4,7 +4,8 @@
 
 # Include source directories and files required for building.
 !go
-!src
+!src/karapace
+!src/schema_registry
 !requirements/*.txt
 !README.rst
 !LICENSE

.github/workflows/container-smoke-test.yml

@@ -8,6 +8,9 @@ on:
 jobs:
   smoke-test-container:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12' ]
     env:
       BUILDKIT_PROGRESS: plain
     steps:
@@ -18,7 +21,7 @@ jobs:
           fetch-depth: 0
 
       - name: Install requirements
-        run: make install
+        run: make install-dev
 
       - name: Resolve Karapace version
         run: |
@@ -32,6 +35,7 @@ jobs:
       - name: Run container
         run: make start-karapace-docker-resources
         env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
           KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
           RUNNER_UID: ${{ env.RUNNER_UID }}
           RUNNER_GID: ${{ env.RUNNER_GID }}

.github/workflows/lint.yml

@@ -41,13 +41,23 @@ jobs:
       with:
         cache: pip
         python-version: '3.12'
-    - name: Install libsnappy-dev
-      run: sudo apt install libsnappy-dev
-    - name: Install requirements and typing requirements
+
+    - name: Install requirements
+      run: make install-dev
+
+    - name: Resolve Karapace version
       run: |
-        python -m venv venv
         source ./venv/bin/activate
-        pip --require-virtualenv install .[typing]
-    - run: |
-        source ./venv/bin/activate
-        mypy src
+        KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
+        echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
+
+    - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+    - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
+
+    - name: Run mypy
+      run: make type-check-mypy-in-docker
+      env:
+        PYTHON_VERSION: 3.12
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}

.github/workflows/tests.yml

@@ -40,14 +40,21 @@ jobs:
       with:
         go-version: '1.21.0'
 
+    - name: Install requirements
+      run: make install-dev
+
     - name: Resolve Karapace version
-      run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV
+      run: |
+        source ./venv/bin/activate
+        KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
+        echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
 
     - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
     - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
 
     - run: make unit-tests-in-docker
       env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
         KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
@@ -56,6 +63,7 @@ jobs:
 
     - run: make e2e-tests-in-docker
       env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
         KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
@@ -64,6 +72,7 @@ jobs:
 
     - run: make integration-tests-in-docker
       env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
         KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}

GNUmakefile

@@ -105,6 +105,10 @@ schema:
 pin-requirements:
 	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"
 
+.PHONY: stop-karapace-docker-resources
+stop-karapace-docker-resources:
+	$(DOCKER_COMPOSE) -f container/compose.yml down -v --remove-orphans
+
 .PHONY: start-karapace-docker-resources
 start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205
 start-karapace-docker-resources:
@@ -121,10 +125,10 @@ unit-tests-in-docker: start-karapace-docker-resources
 
 .PHONY: e2e-tests-in-docker
 e2e-tests-in-docker: export PYTEST_ARGS ?=
-e2e-tests-in-docker: start-karapace-docker-resources
+e2e-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources
 	rm -fr runtime/*
 	sleep 10
-	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/
 	rm -fr runtime/*
 
 .PHONY: integration-tests-in-docker

container/Dockerfile.dev

@@ -1,5 +1,7 @@
+ARG PYTHON_VERSION
+
 # Current versions of avro and zstandard don't yet have wheels for 3.11.
-FROM python:3.10.11-bullseye AS builder
+FROM python:${PYTHON_VERSION}-bullseye AS builder
 
 ARG KARAPACE_VERSION
 ARG RUNNER_UID

container/compose.yml

@@ -124,6 +124,7 @@ services:
       dockerfile: container/Dockerfile.dev
       args:
         KARAPACE_VERSION: $KARAPACE_VERSION
+        PYTHON_VERSION: $PYTHON_VERSION
         RUNNER_UID: $RUNNER_UID
         RUNNER_GID: $RUNNER_GID
     tty: true

mypy.ini

@@ -1,7 +1,7 @@
 [mypy]
 mypy_path = $MYPY_CONFIG_FILE_DIR/stubs
 python_version = 3.10
-packages = karapace
+packages = karapace,schema_registry
 show_error_codes = True
 pretty = True
 warn_redundant_casts = True

pyproject.toml

@@ -113,5 +113,5 @@ include-package-data = true
 version_file = "src/karapace/version.py"
 
 [tool.black]
-target-version = ["py39"]
+target-version = ["py310", "py311"]
 line-length = 125

src/karapace/kafka/consumer.py

@@ -71,7 +71,7 @@ def get_watermark_offsets(
         except KafkaException as exc:
             raise_from_kafkaexception(exc)
 
-    def commit(  # type: ignore[override]
+    def commit(
         self,
         message: Message | None = None,
         offsets: list[TopicPartition] | None = None,
@@ -103,7 +103,7 @@ def committed(self, partitions: list[TopicPartition], timeout: float | None = No
         except KafkaException as exc:
             raise_from_kafkaexception(exc)
 
-    def subscribe(  # type: ignore[override]
+    def subscribe(
         self,
         topics: list[str] | None = None,
         patterns: list[str] | None = None,

src/karapace/schema_reader.py

@@ -421,7 +421,7 @@ def consume_messages(self, msgs: list[Message], watch_offsets: bool) -> None:
                 self.kafka_error_handler.handle_error(location=KafkaErrorLocation.SCHEMA_READER, error=exc)
                 continue  # [non-strict mode]
             except JSONDecodeError as exc:
-                non_bytes_key = msg.key().decode()  # type: ignore[union-attr]
+                non_bytes_key = msg.key().decode()
                 LOG.warning("Invalid JSON in msg.key(): %s at offset %s", non_bytes_key, msg.offset())
                 self.offset = msg.offset()  # Invalid entry shall also move the offset so Karapace makes progress.
                 self.kafka_error_handler.handle_error(location=KafkaErrorLocation.SCHEMA_READER, error=exc)