diff --git a/.dockerignore b/.dockerignore index 4b946a334..0fc564b64 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,8 @@ # Include source directories and files required for building. !go -!src +!src/karapace +!src/schema_registry !requirements/*.txt !README.rst !LICENSE diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml index 25cdc08c9..89d6d4a2d 100644 --- a/.github/workflows/container-smoke-test.yml +++ b/.github/workflows/container-smoke-test.yml @@ -8,6 +8,9 @@ on: jobs: smoke-test-container: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ '3.10', '3.11', '3.12' ] env: BUILDKIT_PROGRESS: plain steps: @@ -18,7 +21,7 @@ jobs: fetch-depth: 0 - name: Install requirements - run: make install + run: make install-dev - name: Resolve Karapace version run: | @@ -32,6 +35,7 @@ jobs: - name: Run container run: make start-karapace-docker-resources env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b8f9b57c3..1de08c44f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,13 +41,23 @@ jobs: with: cache: pip python-version: '3.12' - - name: Install libsnappy-dev - run: sudo apt install libsnappy-dev - - name: Install requirements and typing requirements + + - name: Install requirements + run: make install-dev + + - name: Resolve Karapace version run: | - python -m venv venv source ./venv/bin/activate - pip --require-virtualenv install .[typing] - - run: | - source ./venv/bin/activate - mypy src + KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") + echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV + + - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV + - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV + + - name: Run mypy + run: make type-check-mypy-in-docker + env: + PYTHON_VERSION: 3.12 + KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} + RUNNER_UID: ${{ env.RUNNER_UID }} + RUNNER_GID: ${{ env.RUNNER_GID }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8948dfa4d..df5b5683b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,14 +40,21 @@ jobs: with: go-version: '1.21.0' + - name: Install requirements + run: make install-dev + - name: Resolve Karapace version - run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV + run: | + source ./venv/bin/activate + KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") + echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV - run: make unit-tests-in-docker env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} @@ -56,6 +63,7 @@ jobs: - run: make e2e-tests-in-docker env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} @@ -64,6 +72,7 @@ jobs: - run: make integration-tests-in-docker env: + PYTHON_VERSION: ${{ matrix.python-version }} KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }} RUNNER_UID: ${{ env.RUNNER_UID }} RUNNER_GID: ${{ env.RUNNER_GID }} diff --git a/GNUmakefile b/GNUmakefile index 646a4e271..8e743feec 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -105,6 +105,10 @@ schema: pin-requirements: docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)" +.PHONY: stop-karapace-docker-resources +stop-karapace-docker-resources: + $(DOCKER_COMPOSE) -f container/compose.yml down -v --remove-orphans + .PHONY: start-karapace-docker-resources start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205 start-karapace-docker-resources: @@ -121,10 +125,10 @@ unit-tests-in-docker: start-karapace-docker-resources .PHONY: e2e-tests-in-docker e2e-tests-in-docker: export PYTEST_ARGS ?= -e2e-tests-in-docker: start-karapace-docker-resources +e2e-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources rm -fr runtime/* sleep 10 - $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py + $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/ rm -fr runtime/* .PHONY: integration-tests-in-docker diff --git a/container/Dockerfile.dev b/container/Dockerfile.dev index 3ad6e74b3..826931969 100644 --- a/container/Dockerfile.dev +++ b/container/Dockerfile.dev @@ -1,5 +1,7 @@ +ARG PYTHON_VERSION + # Current versions of avro and zstandard don't yet have wheels for 3.11. -FROM python:3.10.11-bullseye AS builder +FROM python:${PYTHON_VERSION}-bullseye AS builder ARG KARAPACE_VERSION ARG RUNNER_UID diff --git a/container/compose.yml b/container/compose.yml index 5fa01f270..0407f38c1 100644 --- a/container/compose.yml +++ b/container/compose.yml @@ -124,6 +124,7 @@ services: dockerfile: container/Dockerfile.dev args: KARAPACE_VERSION: $KARAPACE_VERSION + PYTHON_VERSION: $PYTHON_VERSION RUNNER_UID: $RUNNER_UID RUNNER_GID: $RUNNER_GID tty: true diff --git a/mypy.ini b/mypy.ini index 523883eb5..5f94f8bc2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,7 +1,7 @@ [mypy] mypy_path = $MYPY_CONFIG_FILE_DIR/stubs python_version = 3.10 -packages = karapace +packages = karapace,schema_registry show_error_codes = True pretty = True warn_redundant_casts = True diff --git a/pyproject.toml b/pyproject.toml index df9a33bda..cd9834ff1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,5 +113,5 @@ include-package-data = true version_file = "src/karapace/version.py" [tool.black] -target-version = ["py39"] +target-version = ["py310", "py311"] line-length = 125 diff --git a/src/karapace/kafka/consumer.py b/src/karapace/kafka/consumer.py index aa6d7ce4d..8623b4178 100644 --- a/src/karapace/kafka/consumer.py +++ b/src/karapace/kafka/consumer.py @@ -71,7 +71,7 @@ def get_watermark_offsets( except KafkaException as exc: raise_from_kafkaexception(exc) - def commit( # type: ignore[override] + def commit( self, message: Message | None = None, offsets: list[TopicPartition] | None = None, @@ -103,7 +103,7 @@ def committed(self, partitions: list[TopicPartition], timeout: float | None = No except KafkaException as exc: raise_from_kafkaexception(exc) - def subscribe( # type: ignore[override] + def subscribe( self, topics: list[str] | None = None, patterns: list[str] | None = None, diff --git a/src/karapace/schema_reader.py b/src/karapace/schema_reader.py index 38af9bde5..bb3bea067 100644 --- a/src/karapace/schema_reader.py +++ b/src/karapace/schema_reader.py @@ -421,7 +421,7 @@ def consume_messages(self, msgs: list[Message], watch_offsets: bool) -> None: self.kafka_error_handler.handle_error(location=KafkaErrorLocation.SCHEMA_READER, error=exc) continue # [non-strict mode] except JSONDecodeError as exc: - non_bytes_key = msg.key().decode() # type: ignore[union-attr] + non_bytes_key = msg.key().decode() LOG.warning("Invalid JSON in msg.key(): %s at offset %s", non_bytes_key, msg.offset()) self.offset = msg.offset() # Invalid entry shall also move the offset so Karapace makes progress. self.kafka_error_handler.handle_error(location=KafkaErrorLocation.SCHEMA_READER, error=exc)