Merge pull request #995 from Aiven-Open/nosahama/fastapi-sr-module-wi…

…th-DI-e2e-tests

feat: fastapi `schema-registry` module with DI and E2E tests

.coveragerc

@@ -1,4 +1,8 @@
 [run]
 branch = true
 relative_files = true
-source = src/karapace
+source = src
+disable_warnings = module-not-measured, no-data-collected
+
+[report]
+skip_empty = true

.dockerignore

@@ -10,7 +10,6 @@
 !LICENSE
 !pyproject.toml
 !setup.py
-!container/start.sh
 !container/healthcheck.py
 
 # Ignore some files in source directories.

.github/workflows/container-smoke-test.yml

@@ -26,14 +26,22 @@ jobs:
           KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
           echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
 
-      - name: Build container
-        run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile .
+      - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+      - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
 
       - name: Run container
-        run: docker compose --file=container/compose.yml up --build --wait --detach
+        run: make start-karapace-docker-resources
+        env:
+          KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+          RUNNER_UID: ${{ env.RUNNER_UID }}
+          RUNNER_GID: ${{ env.RUNNER_GID }}
 
-      - name: Smoke test registry
-        run: bin/smoke-test-registry.sh
+      - name: Smoke test schema registry
+        run: bin/smoke-test-schema-registry.sh
+        env:
+          KARAPACE_PORT: 8081
 
       - name: Smoke test REST proxy
-        run: bin/smoke-test-rest.sh
+        run: bin/smoke-test-rest-proxy.sh
+        env:
+          KARAPACE_PORT: 8082

.github/workflows/tests.yml

@@ -40,22 +40,43 @@ jobs:
       with:
         go-version: '1.21.0'
 
-    - run: make install-dev
-    - run: make unit-tests
+    - name: Resolve Karapace version
+      run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV
+
+    - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+    - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
+
+    - run: make unit-tests-in-docker
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
-    - run: make integration-tests
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4"
+
+    - run: make e2e-tests-in-docker
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4"
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4"
+
+    - run: make integration-tests-in-docker
+      env:
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --random-order --numprocesses 4"
 
     - name: Archive logs
       uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
         name: karapace-integration-test-logs-${{ matrix.python-version }}
         path: /tmp/ci-logs
+
     - name: Archive coverage file
       uses: actions/upload-artifact@v4
       with:

.gitignore

@@ -10,12 +10,13 @@
 __pycache__/
 /build/
 /dist/
-/karapace.egg-info/
+src/karapace.egg-info/
 /karapace-rpm-src.tar
 /kafka_*.tgz
 /kafka_*/
 venv
-/karapace/version.py
+*.so
+src/karapace/version.py
 .run
 .python-version
 .hypothesis/

GNUmakefile

@@ -4,6 +4,8 @@ VENV_DIR ?= $(CURDIR)/venv
 PIP      ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
 PYTHON   ?= python3
 PYTHON_VERSION ?= 3.9
+DOCKER_COMPOSE ?= docker compose
+KARAPACE-CLI   ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli
 
 define PIN_VERSIONS_COMMAND
 pip install pip-tools && \
@@ -102,3 +104,41 @@ schema:
 .PHONY: pin-requirements
 pin-requirements:
 	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"
+
+.PHONY: start-karapace-docker-resources
+start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205
+start-karapace-docker-resources:
+	sudo touch .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12
+	sudo chown ${RUNNER_UID}:${RUNNER_GID} .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12
+	$(DOCKER_COMPOSE) -f container/compose.yml up -d --build --wait --detach
+
+.PHONY: unit-tests-in-docker
+unit-tests-in-docker: export PYTEST_ARGS ?=
+unit-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
+	rm -fr runtime/*
+
+.PHONY: e2e-tests-in-docker
+e2e-tests-in-docker: export PYTEST_ARGS ?=
+e2e-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	sleep 10
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py
+	rm -fr runtime/*
+
+.PHONY: integration-tests-in-docker
+integration-tests-in-docker: export PYTEST_ARGS ?=
+integration-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	sleep 10
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
+	rm -fr runtime/*
+
+.PHONY: type-check-mypy-in-docker
+type-check-mypy-in-docker: start-karapace-docker-resources
+	$(KARAPACE-CLI) $(PYTHON) -m mypy src
+
+.PHONY: cli
+cli: start-karapace-docker-resources
+	$(KARAPACE-CLI) bash

bin/smoke-test-rest.sh → bin/smoke-test-rest-proxy.sh

@@ -3,7 +3,7 @@
 retries=5
 
 for ((i = 0; i <= retries; i++)); do
-    response=$(curl --silent --verbose --fail http://localhost:8082/topics)
+    response=$(curl --silent --verbose --fail "http://localhost:$KARAPACE_PORT/topics")
 
     if [[ $response == '["_schemas","__consumer_offsets"]' ]]; then
         echo "Ok!"

bin/smoke-test-registry.sh → bin/smoke-test-schema-registry.sh

@@ -7,7 +7,7 @@ for ((i = 0; i <= retries; i++)); do
         curl --silent --verbose --fail --request POST \
             --header 'Content-Type: application/vnd.schemaregistry.v1+json' \
             --data '{"schema": "{\"type\": \"record\", \"name\": \"Obj\", \"fields\":[{\"name\": \"age\", \"type\": \"int\"}]}"}' \
-            http://localhost:8081/subjects/test-key/versions
+            "http://localhost:$KARAPACE_PORT/subjects/test-key/versions"
     )
 
     if [[ $response == '{"id":1}' ]]; then

container/Dockerfile

@@ -55,10 +55,6 @@ RUN apt-get update \
 COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"
 
-COPY ./container/start.sh /opt/karapace
-RUN chmod 500 /opt/karapace/start.sh \
- && chown karapace:karapace /opt/karapace/start.sh
-
 COPY ./container/healthcheck.py /opt/karapace
 
 WORKDIR /opt/karapace

container/Dockerfile.dev

@@ -0,0 +1,45 @@
+# Current versions of avro and zstandard don't yet have wheels for 3.11.
+FROM python:3.10.11-bullseye AS builder
+
+ARG KARAPACE_VERSION
+ARG RUNNER_UID
+ARG RUNNER_GID
+
+# Setup files and directories.
+RUN mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace /opt/karapace/coverage \
+&& touch /opt/karapace/coverage/.coverage.3.9 /opt/karapace/coverage/.coverage.3.10 /opt/karapace/coverage/.coverage.3.11 /opt/karapace/coverage/.coverage.3.12 \
+&& chown --recursive "$RUNNER_UID:$RUNNER_GID" /opt/karapace /opt/karapace/coverage /var/log/karapace
+
+# Create, activate, and enforce usage of virtualenv.
+RUN python3 -m venv /opt/karapace/venv
+ENV PATH="/opt/karapace/venv/bin:$PATH"
+ENV PIP_REQUIRE_VIRTUALENV=true
+
+# Install golang needed by extensions
+ENV GO_VERSION=1.21.0
+ENV PATH="/usr/local/go/bin:${PATH}"
+RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
+ && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
+ && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz"
+
+# Install protobuf compiler.
+ARG PROTOBUF_COMPILER_VERSION="3.12.4-1+deb11u1"
+RUN apt-get update \
+ && apt-get install --assume-yes --no-install-recommends \
+    protobuf-compiler=$PROTOBUF_COMPILER_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install Java via openjdk-11
+COPY --from=openjdk:11 /usr/local/openjdk-11 /usr/local/openjdk-11
+ENV JAVA_HOME /usr/local/openjdk-11
+RUN update-alternatives --install /usr/bin/java java /usr/local/openjdk-11/bin/java 1
+
+WORKDIR /opt/karapace
+
+COPY ./requirements /opt/karapace/requirements
+RUN python3 -m pip install -r /opt/karapace/requirements/requirements.txt -r /opt/karapace/requirements/requirements-dev.txt -r /opt/karapace/requirements/requirements-typing.txt
+
+COPY . .
+RUN SETUPTOOLS_SCM_PRETEND_VERSION=$KARAPACE_VERSION python3 -m pip install .
+
+ENV PYTHONPATH="/opt/karapace/src:$PYTHONPATH"

container/compose.yml

@@ -4,7 +4,7 @@ services:
   zookeeper:
     image: confluentinc/cp-zookeeper:latest
     ports:
-      - "2181:2181"
+      - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
@@ -14,16 +14,16 @@ services:
     depends_on:
       - zookeeper
     ports:
-      - "9101:9101"  # JMX
-      - "9092:9092"  # Kafka
+      - 9101:9101  # JMX
+      - 9092:9092  # Kafka
     environment:
       # Listeners:
       # PLAINTEXT_HOST -> Expose kafka to the host network
       # PLAINTEXT -> Used by kafka for inter broker communication / containers
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://karapace-registry:8081
+      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://karapace-schema-registry:8081
       # Metrics:
       KAFKA_JMX_PORT: 9101
       KAFKA_JMX_HOSTNAME: localhost
@@ -54,63 +54,100 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 6000
       KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
 
-  karapace-registry:
+  karapace-schema-registry:
     image: ghcr.io/aiven-open/karapace:develop
     build:
       context: ..
       dockerfile: container/Dockerfile
+      args:
+        KARAPACE_VERSION: $KARAPACE_VERSION
     entrypoint:
-      - /bin/bash
-      - /opt/karapace/start.sh
-      - registry
+      - python3
+      - -m
+      - schema_registry
     depends_on:
       - kafka
     ports:
-      - "8081:8081"
+      - 8081:8081
     environment:
-      KARAPACE_ADVERTISED_HOSTNAME: karapace-registry
+      KARAPACE_KARAPACE_REGISTRY: true
+      KARAPACE_ADVERTISED_HOSTNAME: karapace-schema-registry
       KARAPACE_BOOTSTRAP_URI: kafka:29092
       KARAPACE_PORT: 8081
       KARAPACE_HOST: 0.0.0.0
-      KARAPACE_CLIENT_ID: karapace
-      KARAPACE_GROUP_ID: karapace-registry
-      KARAPACE_MASTER_ELIGIBILITY: "true"
+      KARAPACE_CLIENT_ID: karapace-schema-registry
+      KARAPACE_GROUP_ID: karapace-schema-registry
+      KARAPACE_MASTER_ELIGIBILITY: true
       KARAPACE_TOPIC_NAME: _schemas
-      KARAPACE_LOG_LEVEL: WARNING
+      KARAPACE_LOG_LEVEL: DEBUG
       KARAPACE_COMPATIBILITY: FULL
       KARAPACE_STATSD_HOST: statsd-exporter
       KARAPACE_STATSD_PORT: 8125
       KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
       KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
 
-  karapace-rest:
+  karapace-rest-proxy:
     image: ghcr.io/aiven-open/karapace:develop
     build:
       context: ..
       dockerfile: container/Dockerfile
+      args:
+        KARAPACE_VERSION: $KARAPACE_VERSION
     entrypoint:
-      - /bin/bash
-      - /opt/karapace/start.sh
-      - rest
+      - python3
+      - -m
+      - karapace.karapace_all
     depends_on:
       - kafka
-      - karapace-registry
+      - karapace-schema-registry
     ports:
-      - "8082:8082"
+      - 8082:8082
     environment:
+      KARAPACE_KARAPACE_REST: true
       KARAPACE_PORT: 8082
       KARAPACE_HOST: 0.0.0.0
-      KARAPACE_ADVERTISED_HOSTNAME: karapace-rest
+      KARAPACE_ADVERTISED_HOSTNAME: karapace-rest-proxy
       KARAPACE_BOOTSTRAP_URI: kafka:29092
-      KARAPACE_REGISTRY_HOST: karapace-registry
+      KARAPACE_REGISTRY_HOST: karapace-schema-registry
       KARAPACE_REGISTRY_PORT: 8081
       KARAPACE_ADMIN_METADATA_MAX_AGE: 0
-      KARAPACE_LOG_LEVEL: WARNING
+      KARAPACE_LOG_LEVEL: DEBUG
       KARAPACE_STATSD_HOST: statsd-exporter
       KARAPACE_STATSD_PORT: 8125
       KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
       KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
 
+  karapace-cli:
+    image: ghcr.io/aiven-open/karapace:cli
+    build:
+      context: ..
+      dockerfile: container/Dockerfile.dev
+      args:
+        KARAPACE_VERSION: $KARAPACE_VERSION
+        RUNNER_UID: $RUNNER_UID
+        RUNNER_GID: $RUNNER_GID
+    tty: true
+    depends_on:
+      - kafka
+      - karapace-schema-registry
+      - karapace-rest-proxy
+    volumes:
+    - ../tests:/opt/karapace/tests
+    - ../pytest.ini:/opt/karapace/pytest.ini
+    - ../mypy.ini:/opt/karapace/mypy.ini
+    - ../.flake8:/opt/karapace/.flake8
+    - ../.isort.cfg:/opt/karapace/.isort.cfg
+    - ../.pre-commit-config.yaml:/opt/karapace/.pre-commit-config.yaml
+    - ../.pylintrc:/opt/karapace/.pylintrc
+    - ../.coveragerc:/opt/karapace/.coveragerc
+    - ../.coverage.3.9:/opt/karapace/coverage/.coverage.3.9
+    - ../.coverage.3.10:/opt/karapace/coverage/.coverage.3.10
+    - ../.coverage.3.11:/opt/karapace/coverage/.coverage.3.11
+    - ../.coverage.3.12:/opt/karapace/coverage/.coverage.3.12
+    environment:
+      - COVERAGE_FILE
+      - COVERAGE_RCFILE=/opt/karapace/.coveragerc
+
   prometheus:
     image: prom/prometheus
     volumes: