diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ce27c993..531aa9c2c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,10 +14,9 @@ env: FORCE_COLOR: 1 PIP_PROGRESS_BAR: off PYTHONUNBUFFERED: 1 - KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env jobs: - unit-tests: + tests: runs-on: ubuntu-latest strategy: matrix: @@ -41,8 +40,8 @@ jobs: with: go-version: '1.21.0' - - name: Install requirements - run: make install + - name: Create virtual environment & install requirements + run: make install-dev - name: Resolve Karapace version run: | @@ -53,41 +52,20 @@ jobs: - name: Run containers run: KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} docker compose --file=container/compose.yml up --build --wait --detach - - run: make install-dev - run: make unit-tests-in-docker env: COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" - PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4" - KARAPACE_VERSION=: ${{ env.KARAPACE_VERSION }} - - integration-tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ '3.10', '3.11', '3.12' ] - env: - PYTEST_ADDOPTS: >- - --log-dir=/tmp/ci-logs - --log-file=/tmp/ci-logs/pytest.log - --showlocals - steps: - - uses: actions/checkout@v4 + PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4" - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - cache: pip - python-version: ${{ matrix.python-version }} - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21.0' + - run: make e2e-tests-in-docker + env: + COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" + PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4" - run: make integration-tests env: COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" - PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4" + PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --random-order --numprocesses 4" - name: Archive logs uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 612ad46b2..73ce34070 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,12 @@ __pycache__/ /build/ /dist/ -/karapace.egg-info/ +src/karapace.egg-info/ /karapace-rpm-src.tar /kafka_*.tgz /kafka_*/ venv -/karapace/version.py +src/karapace/version.py .run .python-version .hypothesis/ diff --git a/GNUmakefile b/GNUmakefile index b8305d65a..d515eaa3a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -105,9 +105,21 @@ schema: pin-requirements: docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)" +.PHONY: start-karapace-docker-resources +start-karapace-docker-resources: + $(DOCKER_COMPOSE) -f container/compose.yml up -d + sleep 30 + .PHONY: unit-tests-in-docker unit-tests-in-docker: export PYTEST_ARGS ?= -unit-tests-in-docker: +unit-tests-in-docker: start-karapace-docker-resources rm -fr runtime/* $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ rm -fr runtime/* + +.PHONY: e2e-tests-in-docker +e2e-tests-in-docker: export PYTEST_ARGS ?= +e2e-tests-in-docker: start-karapace-docker-resources + rm -fr runtime/* + $(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/ + rm -fr runtime/* diff --git a/container/karapace.registry.env b/container/karapace.registry.env index cd757a99b..8004c56d8 100644 --- a/container/karapace.registry.env +++ b/container/karapace.registry.env @@ -45,3 +45,6 @@ STATSD_PORT=8125 KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False +HTTP_REQUEST_MAX_SIZE=1048576 +REST_BASE_URI=http://karapace-rest-proxy:8082 +TAGS='{ "app": "karapace-schema-registry" }' diff --git a/container/karapace.rest.env b/container/karapace.rest.env index 3df13f3b2..cf0a854f0 100644 --- a/container/karapace.rest.env +++ b/container/karapace.rest.env @@ -48,4 +48,5 @@ KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False HTTP_REQUEST_MAX_SIZE=1048576 +REST_BASE_URI=http://karapace-rest-proxy:8082 TAGS='{ "app": "karapace-rest-proxy" }' diff --git a/karapace.config.env b/karapace.config.env index 70cf0c616..89cc63764 100644 --- a/karapace.config.env +++ b/karapace.config.env @@ -1,8 +1,9 @@ ACCESS_LOGS_DEBUG=False +ACCESS_LOG_CLASS=aiohttp.web_log.AccessLogger ADVERTISED_HOSTNAME=127.0.0.1 ADVERTISED_PORT=8081 ADVERTISED_PROTOCOL=http -BOOTSTRAP_URI=127.0.0.1:9092 +BOOTSTRAP_URI=kafka:29092 CLIENT_ID=sr-1 COMPATIBILITY=BACKWARD CONNECTIONS_MAX_IDLE_MS=15000 @@ -14,7 +15,7 @@ FETCH_MIN_BYTES=1 GROUP_ID=group_id8357e932 HOST=127.0.0.1 PORT=8081 -REGISTRY_HOST=127.0.0.1 +REGISTRY_HOST=karapace-schema-registry REGISTRY_PORT=8081 REST_AUTHORIZATION=False LOG_HANDLER=stdout @@ -38,8 +39,10 @@ NAME_STRATEGY=topic_name NAME_STRATEGY_VALIDATION=True MASTER_ELECTION_STRATEGY=lowest PROTOBUF_RUNTIME_DIRECTORY=runtime -STATSD_HOST=127.0.0.1 +STATSD_HOST=statsd-exporter STATSD_PORT=8125 KAFKA_SCHEMA_READER_STRICT_MODE=False KAFKA_RETRIABLE_ERRORS_SILENCED=True USE_PROTOBUF_FORMATTER=False +REST_BASE_URI=http://karapace-rest-proxy:8082 +TAGS='{ "app": "karapace" }' diff --git a/mypy.ini b/mypy.ini index c4ef8efd1..30d56b0bc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -15,7 +15,7 @@ warn_no_return = True warn_unreachable = True strict_equality = True -[mypy-karapace.schema_registry_apis] +[mypy-schema_registry.schema_registry_apis] ignore_errors = True [mypy-karapace.compatibility.jsonschema.checks] diff --git a/pyproject.toml b/pyproject.toml index f1f9016cb..6512d9d23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ dependencies = [ "zstandard", "prometheus-client == 0.20.0", "yarl == 1.12.1", + "opentelemetry-api == 1.28.2", + "opentelemetry-sdk == 1.28.2", + "opentelemetry-instrumentation-fastapi == 0.49b2", + "dependency-injector == 4.43.0", # Patched dependencies # @@ -105,6 +109,12 @@ typing = [ [tool.setuptools] include-package-data = true +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +karapace = ["*.yaml"] + [tool.setuptools_scm] version_file = "src/karapace/version.py" diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 9848f80e0..6c6e4a9e7 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -38,20 +43,35 @@ cachetools==5.3.3 certifi==2024.8.30 # via # geventhttpclient + # httpcore + # httpx # requests # sentry-sdk charset-normalizer==3.4.0 # via requests click==8.1.7 - # via flask + # via + # flask + # typer + # uvicorn configargparse==1.7 # via locust confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) -coverage[toml]==7.6.4 +coverage[toml]==7.6.8 # via pytest-cov cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via # anyio @@ -61,9 +81,13 @@ execnet==2.1.1 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi filelock==3.16.1 # via karapace (/karapace/pyproject.toml) -flask==3.0.3 +flask==3.1.0 # via # flask-cors # flask-login @@ -80,19 +104,31 @@ gevent==24.11.1 # via # geventhttpclient # locust -geventhttpclient==2.3.1 +geventhttpclient==2.3.3 # via locust greenlet==3.1.1 # via gevent -hypothesis==6.118.8 +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi +hypothesis==6.119.4 # via karapace (/karapace/pyproject.toml) idna==3.10 # via # anyio + # email-validator + # httpx # requests # yarl importlib-metadata==8.5.0 - # via flask + # via opentelemetry-api iniconfig==2.0.0 # via pytest isodate==0.7.2 @@ -100,12 +136,14 @@ isodate==0.7.2 itsdangerous==2.2.0 # via flask jinja2==3.1.4 - # via flask + # via + # fastapi + # flask jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 # via jsonschema -locust==2.32.2 +locust==2.32.3 # via karapace (/karapace/pyproject.toml) lz4==4.3.3 # via karapace (/karapace/pyproject.toml) @@ -123,11 +161,40 @@ multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 # via # aiokafka + # opentelemetry-instrumentation # pytest pdbpp==0.10.3 # via karapace (/karapace/pyproject.toml) @@ -142,11 +209,15 @@ psutil==6.1.0 # karapace (/karapace/pyproject.toml) # locust # pytest-xdist +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via # pdbpp # rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) pyrepl==0.9.0 # via fancycompleter @@ -167,8 +238,14 @@ pytest-xdist[psutil]==3.6.1 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn pyzmq==26.2.0 # via locust referencing==0.35.1 @@ -180,19 +257,29 @@ requests==2.32.3 # karapace (/karapace/pyproject.toml) # locust rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx sortedcontainers==2.4.0 # via hypothesis +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) tomli==2.1.0 @@ -200,12 +287,20 @@ tomli==2.1.0 # coverage # locust # pytest +typer==0.13.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # locust # multidict + # opentelemetry-sdk + # pydantic + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 @@ -213,8 +308,18 @@ urllib3==2.2.3 # geventhttpclient # requests # sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn werkzeug==3.1.3 # via # flask @@ -222,6 +327,10 @@ werkzeug==3.1.3 # locust wmctrl==0.5 # via pdbpp +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index aef63ee86..beab6babb 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -30,23 +35,60 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) certifi==2024.8.30 - # via sentry-sdk + # via + # httpcore + # httpx + # sentry-sdk +click==8.1.7 + # via + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -55,6 +97,8 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 @@ -65,43 +109,95 @@ mypy==1.13.0 # via karapace (/karapace/pyproject.toml) mypy-extensions==1.0.0 # via mypy -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications # types-jsonschema rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) tomli==2.1.0 # via mypy +typer==0.13.1 + # via fastapi-cli types-cachetools==5.5.0.20240820 # via karapace (/karapace/pyproject.toml) types-jsonschema==4.23.0.20240813 @@ -111,20 +207,42 @@ types-protobuf==3.20.4.6 typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict # mypy + # opentelemetry-sdk + # pydantic + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 # via sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5bb9cf22e..ccdc011c4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make pin-requirements @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -29,22 +34,60 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c # via karapace (/karapace/pyproject.toml) cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) +certifi==2024.8.30 + # via + # httpcore + # httpx +click==8.1.7 + # via + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -53,58 +96,134 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 # via # aiohttp # yarl -networkx==3.2.1 +networkx==3.4.2 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) +typer==0.13.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict + # opentelemetry-sdk + # pydantic + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/src/karapace.egg-info/top_level.txt b/src/karapace.egg-info/top_level.txt new file mode 100644 index 000000000..a3a1d0ab2 --- /dev/null +++ b/src/karapace.egg-info/top_level.txt @@ -0,0 +1,3 @@ +karapace +protopacelib +schema_registry diff --git a/src/karapace/dependencies/config_dependency.py b/src/karapace/dependencies/config_dependency.py deleted file mode 100644 index 9c299b725..000000000 --- a/src/karapace/dependencies/config_dependency.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import Depends -from karapace.config import Config -from typing import Annotated - -import os - -env_file = os.environ.get("KARAPACE_DOTENV", None) - - -class ConfigDependencyManager: - CONFIG = Config(_env_file=env_file, _env_file_encoding="utf-8") - - @classmethod - def get_config(cls) -> Config: - return ConfigDependencyManager.CONFIG - - -ConfigDep = Annotated[Config, Depends(ConfigDependencyManager.get_config)] diff --git a/src/karapace/dependencies/controller_dependency.py b/src/karapace/dependencies/controller_dependency.py deleted file mode 100644 index e056b52c2..000000000 --- a/src/karapace/dependencies/controller_dependency.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - - -from fastapi import Depends -from karapace.dependencies.config_dependency import ConfigDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.dependencies.stats_dependeny import StatsDep -from karapace.schema_registry_apis import KarapaceSchemaRegistryController -from typing import Annotated - - -async def get_controller( - config: ConfigDep, - stats: StatsDep, - schema_registry: SchemaRegistryDep, -) -> KarapaceSchemaRegistryController: - return KarapaceSchemaRegistryController(config=config, schema_registry=schema_registry, stats=stats) - - -KarapaceSchemaRegistryControllerDep = Annotated[KarapaceSchemaRegistryController, Depends(get_controller)] diff --git a/src/karapace/dependencies/forward_client_dependency.py b/src/karapace/dependencies/forward_client_dependency.py deleted file mode 100644 index 57459c371..000000000 --- a/src/karapace/dependencies/forward_client_dependency.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import Depends -from karapace.forward_client import ForwardClient -from typing import Annotated - -FORWARD_CLIENT: ForwardClient | None = None - - -def get_forward_client() -> ForwardClient: - global FORWARD_CLIENT - if not FORWARD_CLIENT: - FORWARD_CLIENT = ForwardClient() - return FORWARD_CLIENT - - -ForwardClientDep = Annotated[ForwardClient, Depends(get_forward_client)] diff --git a/src/karapace/dependencies/schema_registry_dependency.py b/src/karapace/dependencies/schema_registry_dependency.py deleted file mode 100644 index 68d9b0700..000000000 --- a/src/karapace/dependencies/schema_registry_dependency.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import Depends -from karapace.dependencies.config_dependency import ConfigDependencyManager -from karapace.schema_registry import KarapaceSchemaRegistry -from typing import Annotated - - -class SchemaRegistryDependencyManager: - SCHEMA_REGISTRY: KarapaceSchemaRegistry | None = None - - @classmethod - async def get_schema_registry(cls) -> KarapaceSchemaRegistry: - if not SchemaRegistryDependencyManager.SCHEMA_REGISTRY: - SchemaRegistryDependencyManager.SCHEMA_REGISTRY = KarapaceSchemaRegistry( - config=ConfigDependencyManager.get_config() - ) - return SchemaRegistryDependencyManager.SCHEMA_REGISTRY - - -SchemaRegistryDep = Annotated[KarapaceSchemaRegistry, Depends(SchemaRegistryDependencyManager.get_schema_registry)] diff --git a/src/karapace/dependencies/stats_dependeny.py b/src/karapace/dependencies/stats_dependeny.py deleted file mode 100644 index 98c116dac..000000000 --- a/src/karapace/dependencies/stats_dependeny.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - - -from fastapi import Depends -from karapace.dependencies.config_dependency import ConfigDependencyManager -from karapace.statsd import StatsClient -from typing import Annotated - - -class StatsDependencyManager: - STATS_CLIENT: StatsClient | None = None - - @classmethod - def get_stats(cls) -> StatsClient: - if not StatsDependencyManager.STATS_CLIENT: - StatsDependencyManager.STATS_CLIENT = StatsClient(config=ConfigDependencyManager.get_config()) - return StatsDependencyManager.STATS_CLIENT - - -StatsDep = Annotated[StatsClient, Depends(StatsDependencyManager.get_stats)] diff --git a/src/karapace/instrumentation/prometheus.py b/src/karapace/instrumentation/prometheus.py index 1336b4ab0..90d260057 100644 --- a/src/karapace/instrumentation/prometheus.py +++ b/src/karapace/instrumentation/prometheus.py @@ -22,6 +22,7 @@ class PrometheusInstrumentation: METRICS_ENDPOINT_PATH: Final[str] = "/metrics" + CONTENT_TYPE_LATEST: Final[str] = "text/plain; version=0.0.4; charset=utf-8" START_TIME_REQUEST_KEY: Final[str] = "start_time" registry: Final[CollectorRegistry] = CollectorRegistry() diff --git a/src/karapace/protobuf/io.py b/src/karapace/protobuf/io.py index 36c76e491..89cdd26f1 100644 --- a/src/karapace/protobuf/io.py +++ b/src/karapace/protobuf/io.py @@ -97,7 +97,7 @@ def get_protobuf_class_instance( class_name: str, cfg: Config, ) -> _ProtobufModel: - directory = Path(cfg["protobuf_runtime_directory"]) + directory = Path(cfg.protobuf_runtime_directory) deps_list = crawl_dependencies(schema) root_class_name = "" for value in deps_list.values(): diff --git a/src/karapace/routers/compatibility_router.py b/src/karapace/routers/compatibility_router.py deleted file mode 100644 index 0db406d2a..000000000 --- a/src/karapace/routers/compatibility_router.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.errors import unauthorized -from karapace.routers.requests import CompatibilityCheckResponse, SchemaRequest -from karapace.typing import Subject - -compatibility_router = APIRouter( - prefix="/compatibility", - tags=["compatibility"], - responses={404: {"description": "Not found"}}, -) - - -@compatibility_router.post("/subjects/{subject}/versions/{version}", response_model_exclude_none=True) -async def compatibility_post( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, # TODO support actual Version object - schema_request: SchemaRequest, -) -> CompatibilityCheckResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.compatibility_check(subject=subject, schema_request=schema_request, version=version) diff --git a/src/karapace/routers/config_router.py b/src/karapace/routers/config_router.py deleted file mode 100644 index a83f24f60..000000000 --- a/src/karapace/routers/config_router.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter, Request -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.dependencies.forward_client_dependency import ForwardClientDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.routers.errors import no_primary_url_error, unauthorized -from karapace.routers.requests import CompatibilityLevelResponse, CompatibilityRequest, CompatibilityResponse -from karapace.typing import Subject - -config_router = APIRouter( - prefix="/config", - tags=["config"], - responses={404: {"description": "Not found"}}, -) - - -@config_router.get("") -async def config_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, -) -> CompatibilityLevelResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, "Config:"): - raise unauthorized() - - return await controller.config_get() - - -@config_router.put("") -async def config_put( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - compatibility_level_request: CompatibilityRequest, -) -> CompatibilityResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, "Config:"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.config_set(compatibility_level_request=compatibility_level_request) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@config_router.get("/{subject}") -async def config_get_subject( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - defaultToGlobal: bool = False, -) -> CompatibilityLevelResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.config_subject_get(subject=subject, default_to_global=defaultToGlobal) - - -@config_router.put("/{subject}") -async def config_set_subject( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - compatibility_level_request: CompatibilityRequest, -) -> CompatibilityResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.config_subject_set(subject=subject, compatibility_level_request=compatibility_level_request) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@config_router.delete("/{subject}") -async def config_delete_subject( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, -) -> CompatibilityResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.config_subject_delete(subject=subject) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) diff --git a/src/karapace/routers/errors.py b/src/karapace/routers/errors.py deleted file mode 100644 index a16c9797a..000000000 --- a/src/karapace/routers/errors.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from enum import Enum, unique -from fastapi import HTTPException, status -from fastapi.exceptions import RequestValidationError - - -@unique -class SchemaErrorCodes(Enum): - HTTP_BAD_REQUEST = status.HTTP_400_BAD_REQUEST - HTTP_NOT_FOUND = status.HTTP_404_NOT_FOUND - HTTP_CONFLICT = status.HTTP_409_CONFLICT - HTTP_UNPROCESSABLE_ENTITY = status.HTTP_422_UNPROCESSABLE_ENTITY - HTTP_INTERNAL_SERVER_ERROR = status.HTTP_500_INTERNAL_SERVER_ERROR - SUBJECT_NOT_FOUND = 40401 - VERSION_NOT_FOUND = 40402 - SCHEMA_NOT_FOUND = 40403 - SUBJECT_SOFT_DELETED = 40404 - SUBJECT_NOT_SOFT_DELETED = 40405 - SCHEMAVERSION_SOFT_DELETED = 40406 - SCHEMAVERSION_NOT_SOFT_DELETED = 40407 - SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_ERROR_CODE = 40408 - INVALID_VERSION_ID = 42202 - INVALID_COMPATIBILITY_LEVEL = 42203 - INVALID_SCHEMA = 42201 - INVALID_SUBJECT = 42208 - SCHEMA_TOO_LARGE_ERROR_CODE = 42209 - REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 - REFERENCE_EXISTS = 42206 - NO_MASTER_ERROR = 50003 - - -class KarapaceValidationError(RequestValidationError): - def __init__(self, error_code: int, error: str): - super().__init__(errors=[], body=error) - self.error_code = error_code - - -def no_primary_url_error() -> HTTPException: - return HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail={ - "error_code": SchemaErrorCodes.NO_MASTER_ERROR, - "message": "Error while forwarding the request to the master.", - }, - ) - - -def unauthorized() -> HTTPException: - return HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail={"message": "Forbidden"}, - ) diff --git a/src/karapace/routers/health_router.py b/src/karapace/routers/health_router.py deleted file mode 100644 index 950e08cfc..000000000 --- a/src/karapace/routers/health_router.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter, HTTPException, status -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from pydantic import BaseModel - - -class HealthStatus(BaseModel): - schema_registry_ready: bool - schema_registry_startup_time_sec: float - schema_registry_reader_current_offset: int - schema_registry_reader_highest_offset: int - schema_registry_is_primary: bool | None - schema_registry_is_primary_eligible: bool - schema_registry_primary_url: str | None - schema_registry_coordinator_running: bool - schema_registry_coordinator_generation_id: int - - -class HealthCheck(BaseModel): - status: HealthStatus - healthy: bool - - -health_router = APIRouter( - prefix="/_health", - tags=["health"], - responses={404: {"description": "Not found"}}, -) - - -@health_router.get("") -async def health( - schema_registry: SchemaRegistryDep, -) -> HealthCheck: - starttime = 0.0 - if schema_registry.schema_reader.ready(): - starttime = schema_registry.schema_reader.last_check - schema_registry.schema_reader.start_time - - cs = schema_registry.mc.get_coordinator_status() - - health_status = HealthStatus( - schema_registry_ready=schema_registry.schema_reader.ready(), - schema_registry_startup_time_sec=starttime, - schema_registry_reader_current_offset=schema_registry.schema_reader.offset, - schema_registry_reader_highest_offset=schema_registry.schema_reader.highest_offset(), - schema_registry_is_primary=cs.is_primary, - schema_registry_is_primary_eligible=cs.is_primary_eligible, - schema_registry_primary_url=cs.primary_url, - schema_registry_coordinator_running=cs.is_running, - schema_registry_coordinator_generation_id=cs.group_generation_id, - ) - # if self._auth is not None: - # resp["schema_registry_authfile_timestamp"] = self._auth.authfile_last_modified - - if not await schema_registry.schema_reader.is_healthy(): - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) - - return HealthCheck(status=health_status, healthy=True) diff --git a/src/karapace/routers/mode_router.py b/src/karapace/routers/mode_router.py deleted file mode 100644 index d8c98363a..000000000 --- a/src/karapace/routers/mode_router.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.errors import unauthorized -from karapace.typing import Subject - -mode_router = APIRouter( - prefix="/mode", - tags=["mode"], - responses={404: {"description": "Not found"}}, -) - - -@mode_router.get("") -async def mode_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, -): - if authorizer and not authorizer.check_authorization(user, Operation.Read, "Config:"): - raise unauthorized() - - return await controller.get_global_mode() - - -@mode_router.get("/{subject}") -async def mode_get_subject( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, -): - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.get_subject_mode(subject=subject) diff --git a/src/karapace/routers/requests.py b/src/karapace/routers/requests.py deleted file mode 100644 index 8400f629d..000000000 --- a/src/karapace/routers/requests.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from karapace.routers.errors import KarapaceValidationError -from karapace.schema_type import SchemaType -from karapace.typing import Subject -from pydantic import BaseModel, Field, validator -from typing import Any - - -class SchemaReference(BaseModel): - name: str - subject: Subject - version: int - - -class SchemaRequest(BaseModel): - schema_str: str = Field(alias="schema") - schema_type: SchemaType = Field(alias="schemaType", default=SchemaType.AVRO) - references: list[SchemaReference] | None = None - metadata: Any | None - ruleSet: Any | None - - class Config: - extra = "forbid" - - @validator("schema_str") - def validate_schema(cls, schema_str: str) -> str: - if not schema_str and not schema_str.strip(): - raise KarapaceValidationError( - error_code=42201, - error="Empty schema", - ) - return schema_str - - -class SchemaResponse(BaseModel): - subject: Subject - version: int - schema_id: int = Field(alias="id") - schema_str: str = Field(alias="schema") - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - - -class SchemasResponse(BaseModel): - schema_str: str = Field(alias="schema") - subjects: list[Subject] | None = None - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - references: list[Any] | None = None # TODO: typing - maxId: int | None = None - - -class SchemaListingItem(BaseModel): - subject: Subject - schema_str: str = Field(alias="schema") - version: int - schema_id: int = Field(alias="id") - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - references: list[Any] | None - - -class SchemaIdResponse(BaseModel): - schema_id: int = Field(alias="id") - - -class CompatibilityRequest(BaseModel): - compatibility: str - - -class CompatibilityResponse(BaseModel): - compatibility: str - - -class CompatibilityLevelResponse(BaseModel): - compatibility_level: str = Field(alias="compatibilityLevel") - - -class CompatibilityCheckResponse(BaseModel): - is_compatible: bool - messages: list[str] | None = None - - -class ModeResponse(BaseModel): - mode: str - - -class SubjectVersion(BaseModel): - subject: Subject - version: int - - -class SubjectSchemaVersionResponse(BaseModel): - subject: Subject - version: int - schema_id: int = Field(alias="id") - schema_str: str = Field(alias="schema") - references: list[Any] | None = None - schema_type: SchemaType | None = Field(alias="schemaType", default=None) - compatibility: str | None = None diff --git a/src/karapace/routers/root_router.py b/src/karapace/routers/root_router.py deleted file mode 100644 index 6bec6cb9c..000000000 --- a/src/karapace/routers/root_router.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter - -root_router = APIRouter( - tags=["root"], - responses={404: {"description": "Not found"}}, -) - - -@root_router.get("/") -async def root() -> dict: - return {} diff --git a/src/karapace/routers/schemas_router.py b/src/karapace/routers/schemas_router.py deleted file mode 100644 index c06cd4a48..000000000 --- a/src/karapace/routers/schemas_router.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.requests import SchemaListingItem, SchemasResponse, SubjectVersion - -schemas_router = APIRouter( - prefix="/schemas", - tags=["schemas"], - responses={404: {"description": "Not found"}}, -) - - -# TODO is this needed? Is this actually the ids/schema/id/schema?? -@schemas_router.get("") -async def schemas_get_list( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - deleted: bool = False, - latestOnly: bool = False, -) -> list[SchemaListingItem]: - return await controller.schemas_list( - deleted=deleted, - latest_only=latestOnly, - user=user, - authorizer=authorizer, - ) - - -@schemas_router.get("/ids/{schema_id}", response_model_exclude_none=True) -async def schemas_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - schema_id: str, # TODO: type to actual type - includeSubjects: bool = False, # TODO: include subjects? - fetchMaxId: bool = False, # TODO: fetch max id? - format: str = "", -) -> SchemasResponse: - return await controller.schemas_get( - schema_id=schema_id, - include_subjects=includeSubjects, - fetch_max_id=fetchMaxId, - format_serialized=format, - user=user, - authorizer=authorizer, - ) - - -# @schemas_router.get("/ids/{schema_id}/schema") -# async def schemas_get_only_id( -# controller: KarapaceSchemaRegistryControllerDep, -# ) -> SchemasResponse: -# # TODO retrieve by id only schema -# return await controller.schemas_get() - - -@schemas_router.get("/ids/{schema_id}/versions") -async def schemas_get_versions( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - schema_id: str, - deleted: bool = False, -) -> list[SubjectVersion]: - return await controller.schemas_get_versions( - schema_id=schema_id, - deleted=deleted, - user=user, - authorizer=authorizer, - ) - - -@schemas_router.get("/types") -async def schemas_get_subjects_list( - controller: KarapaceSchemaRegistryControllerDep, -) -> list[str]: - return await controller.schemas_types() diff --git a/src/karapace/routers/subjects_router.py b/src/karapace/routers/subjects_router.py deleted file mode 100644 index 9bde67743..000000000 --- a/src/karapace/routers/subjects_router.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details -""" - -from fastapi import APIRouter, Request -from karapace.auth.auth import Operation -from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep -from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.dependencies.forward_client_dependency import ForwardClientDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.routers.errors import no_primary_url_error, unauthorized -from karapace.routers.requests import SchemaIdResponse, SchemaRequest, SchemaResponse, SubjectSchemaVersionResponse -from karapace.typing import Subject - -import logging - -LOG = logging.getLogger(__name__) - - -subjects_router = APIRouter( - prefix="/subjects", - tags=["subjects"], - responses={404: {"description": "Not found"}}, -) - - -@subjects_router.get("") -async def subjects_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - deleted: bool = False, -) -> list[str]: - return await controller.subjects_list( - deleted=deleted, - user=user, - authorizer=authorizer, - ) - - -@subjects_router.post("/{subject}", response_model_exclude_none=True) -async def subjects_subject_post( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - schema_request: SchemaRequest, - deleted: bool = False, - normalize: bool = False, -) -> SchemaResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subjects_schema_post( - subject=subject, - schema_request=schema_request, - deleted=deleted, - normalize=normalize, - ) - - -@subjects_router.delete("/{subject}") -async def subjects_subject_delete( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - permanent: bool = False, -) -> list[int]: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.subject_delete(subject=subject, permanent=permanent) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@subjects_router.post("/{subject}/versions") -async def subjects_subject_versions_post( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - schema_request: SchemaRequest, - normalize: bool = False, -) -> SchemaIdResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - # TODO: split the functionality so primary error and forwarding can be handled here - # and local/primary write is in controller. - return await controller.subject_post( - subject=subject, - schema_request=schema_request, - normalize=normalize, - forward_client=forward_client, - request=request, - ) - - -@subjects_router.get("/{subject}/versions") -async def subjects_subject_versions_list( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - deleted: bool = False, -) -> list[int]: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_versions_list(subject=subject, deleted=deleted) - - -@subjects_router.get("/{subject}/versions/{version}", response_model_exclude_none=True) -async def subjects_subject_version_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, - deleted: bool = False, -) -> SubjectSchemaVersionResponse: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_version_get(subject=subject, version=version, deleted=deleted) - - -@subjects_router.delete("/{subject}/versions/{version}") -async def subjects_subject_version_delete( - request: Request, - controller: KarapaceSchemaRegistryControllerDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, - permanent: bool = False, -) -> int: - if authorizer and not authorizer.check_authorization(user, Operation.Write, f"Subject:{subject}"): - raise unauthorized() - - i_am_primary, primary_url = await schema_registry.get_master() - if i_am_primary: - return await controller.subject_version_delete(subject=subject, version=version, permanent=permanent) - elif not primary_url: - raise no_primary_url_error() - else: - return await forward_client.forward_request_remote(request=request, primary_url=primary_url) - - -@subjects_router.get("/{subject}/versions/{version}/schema") -async def subjects_subject_version_schema_get( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, -) -> dict: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_version_schema_get(subject=subject, version=version) - - -@subjects_router.get("/{subject}/versions/{version}/referencedby") -async def subjects_subject_version_referenced_by( - controller: KarapaceSchemaRegistryControllerDep, - user: CurrentUserDep, - authorizer: AuthenticatorAndAuthorizerDep, - subject: Subject, - version: str, -) -> list[int]: - if authorizer and not authorizer.check_authorization(user, Operation.Read, f"Subject:{subject}"): - raise unauthorized() - - return await controller.subject_version_referencedby_get(subject=subject, version=version) diff --git a/src/schema_registry/__main__.py b/src/schema_registry/__main__.py index 0663bf774..d6df9866b 100644 --- a/src/schema_registry/__main__.py +++ b/src/schema_registry/__main__.py @@ -11,6 +11,7 @@ import schema_registry.routers.compatibility import schema_registry.routers.config import schema_registry.routers.health +import schema_registry.routers.master_availability import schema_registry.routers.metrics import schema_registry.routers.mode import schema_registry.routers.schemas @@ -42,6 +43,7 @@ schema_registry.routers.config, schema_registry.routers.compatibility, schema_registry.routers.mode, + schema_registry.routers.master_availability, ] ) diff --git a/src/schema_registry/routers/health.py b/src/schema_registry/routers/health.py index df3a8822f..ce1ef9181 100644 --- a/src/schema_registry/routers/health.py +++ b/src/schema_registry/routers/health.py @@ -40,13 +40,13 @@ async def health( schema_registry: KarapaceSchemaRegistry = Depends(Provide[SchemaRegistryContainer.karapace_container.schema_registry]), ) -> HealthCheck: starttime = 0.0 - if schema_registry.schema_reader.ready: + if schema_registry.schema_reader.ready(): starttime = schema_registry.schema_reader.last_check - schema_registry.schema_reader.start_time cs = schema_registry.mc.get_coordinator_status() health_status = HealthStatus( - schema_registry_ready=schema_registry.schema_reader.ready, + schema_registry_ready=schema_registry.schema_reader.ready(), schema_registry_startup_time_sec=starttime, schema_registry_reader_current_offset=schema_registry.schema_reader.offset, schema_registry_reader_highest_offset=schema_registry.schema_reader.highest_offset(), diff --git a/src/karapace/routers/master_available_router.py b/src/schema_registry/routers/master_availability.py similarity index 71% rename from src/karapace/routers/master_available_router.py rename to src/schema_registry/routers/master_availability.py index e55389f42..55e792275 100644 --- a/src/karapace/routers/master_available_router.py +++ b/src/schema_registry/routers/master_availability.py @@ -3,13 +3,14 @@ See LICENSE for details """ -from fastapi import APIRouter, HTTPException, Request, Response, status +from dependency_injector.wiring import inject, Provide +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from karapace.config import LOG -from karapace.dependencies.config_dependency import ConfigDep -from karapace.dependencies.forward_client_dependency import ForwardClientDep -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep +from karapace.config import Config +from karapace.forward_client import ForwardClient +from karapace.schema_registry import KarapaceSchemaRegistry from pydantic import BaseModel +from schema_registry.container import SchemaRegistryContainer from typing import Final import json @@ -33,12 +34,13 @@ class MasterAvailabilityResponse(BaseModel): @master_availability_router.get("") -async def master_available( - config: ConfigDep, - schema_registry: SchemaRegistryDep, - forward_client: ForwardClientDep, +@inject +async def master_availability( request: Request, response: Response, + config: Config = Depends(Provide[SchemaRegistryContainer.karapace_container.config]), + forward_client: ForwardClient = Depends(Provide[SchemaRegistryContainer.karapace_container.forward_client]), + schema_registry: KarapaceSchemaRegistry = Depends(Provide[SchemaRegistryContainer.karapace_container.schema_registry]), ) -> MasterAvailabilityResponse: are_we_master, master_url = await schema_registry.get_master() LOG.info("are master %s, master url %s", are_we_master, master_url) diff --git a/src/schema_registry/routers/setup.py b/src/schema_registry/routers/setup.py index fe0b6be9b..663639583 100644 --- a/src/schema_registry/routers/setup.py +++ b/src/schema_registry/routers/setup.py @@ -7,6 +7,7 @@ from schema_registry.routers.compatibility import compatibility_router from schema_registry.routers.config import config_router from schema_registry.routers.health import health_router +from schema_registry.routers.master_availability import master_availability_router from schema_registry.routers.metrics import metrics_router from schema_registry.routers.mode import mode_router from schema_registry.routers.root import root_router @@ -23,3 +24,4 @@ def setup_routers(app: FastAPI) -> None: app.include_router(schemas_router) app.include_router(subjects_router) app.include_router(metrics_router) + app.include_router(master_availability_router) diff --git a/src/schema_registry/schema_registry_apis.py b/src/schema_registry/schema_registry_apis.py index cc9a01bb2..13f6bb8f2 100644 --- a/src/schema_registry/schema_registry_apis.py +++ b/src/schema_registry/schema_registry_apis.py @@ -65,9 +65,6 @@ class KarapaceSchemaRegistryController: def __init__(self, config: Config, schema_registry: KarapaceSchemaRegistry, stats: StatsClient) -> None: # super().__init__(config=config, not_ready_handler=self._forward_if_not_ready_to_serve) - print("+++++++++========") - print(schema_registry) - self.config = config self._process_start_time = time.monotonic() self.stats = stats @@ -219,8 +216,6 @@ def _has_subject_with_id() -> bool: ) schema = self.schema_registry.schemas_get(parsed_schema_id, fetch_max_id=fetch_max_id) - print("+++++++++========") - print(schema) if not schema: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/src/schema_registry/user.py b/src/schema_registry/user.py index 16cd55705..921f39ea4 100644 --- a/src/schema_registry/user.py +++ b/src/schema_registry/user.py @@ -13,14 +13,9 @@ @inject async def get_current_user( - credentials: Annotated[HTTPBasicCredentials, Depends(HTTPBasic())], + credentials: Annotated[HTTPBasicCredentials, Depends(HTTPBasic(auto_error=False))], authorizer: AuthenticatorAndAuthorizer = Depends(Provide[SchemaRegistryContainer.karapace_container.authorizer]), ) -> User: - import logging - - logging.info("get_current_user ++++++++++++=============") - logging.info(f"credentials: {credentials}") - logging.info(f"authorizer: {authorizer}") if authorizer and not credentials: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 000000000..f53be7121 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 000000000..050ced269 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,131 @@ +""" +karapace - conftest + +Copyright (c) 2023 Aiven Ltd +See LICENSE for details +""" +from __future__ import annotations + +from _pytest.fixtures import SubRequest +from aiohttp import BasicAuth +from collections.abc import AsyncGenerator, Iterator +from confluent_kafka.admin import NewTopic +from karapace.client import Client +from karapace.config import KARAPACE_BASE_CONFIG_YAML_PATH +from karapace.container import KarapaceContainer +from karapace.kafka.admin import KafkaAdminClient +from karapace.kafka.consumer import AsyncKafkaConsumer, KafkaConsumer +from karapace.kafka.producer import AsyncKafkaProducer, KafkaProducer +from tests.integration.utils.cluster import RegistryDescription, RegistryEndpoint +from tests.integration.utils.kafka_server import KafkaServers + +import asyncio +import pytest +import secrets + + +@pytest.fixture(scope="session", name="basic_auth") +def fixture_basic_auth() -> BasicAuth: + return BasicAuth("test", "test") + + +@pytest.fixture(name="karapace_container", scope="session") +def fixture_karapace_container() -> KarapaceContainer: + container = KarapaceContainer() + container.base_config.from_yaml(KARAPACE_BASE_CONFIG_YAML_PATH, envs_required=True, required=True) + return container + + +@pytest.fixture(scope="session", name="kafka_servers") +def fixture_kafka_server(karapace_container: KarapaceContainer) -> Iterator[KafkaServers]: + yield KafkaServers([karapace_container.config().bootstrap_uri]) + + +@pytest.fixture(scope="function", name="producer") +def fixture_producer(kafka_servers: KafkaServers) -> Iterator[KafkaProducer]: + yield KafkaProducer(bootstrap_servers=kafka_servers.bootstrap_servers) + + +@pytest.fixture(scope="function", name="admin_client") +def fixture_admin(kafka_servers: KafkaServers) -> Iterator[KafkaAdminClient]: + yield KafkaAdminClient(bootstrap_servers=kafka_servers.bootstrap_servers) + + +@pytest.fixture(scope="function", name="consumer") +def fixture_consumer( + kafka_servers: KafkaServers, +) -> Iterator[KafkaConsumer]: + consumer = KafkaConsumer( + bootstrap_servers=kafka_servers.bootstrap_servers, + auto_offset_reset="earliest", + enable_auto_commit=False, + topic_metadata_refresh_interval_ms=200, # Speed things up for consumer tests to discover topics, etc. + ) + try: + yield consumer + finally: + consumer.close() + + +@pytest.fixture(scope="function", name="asyncproducer") +async def fixture_asyncproducer( + kafka_servers: KafkaServers, + loop: asyncio.AbstractEventLoop, +) -> AsyncGenerator[AsyncKafkaProducer, None]: + asyncproducer = AsyncKafkaProducer(bootstrap_servers=kafka_servers.bootstrap_servers, loop=loop) + await asyncproducer.start() + yield asyncproducer + await asyncproducer.stop() + + +@pytest.fixture(scope="function", name="asyncconsumer") +async def fixture_asyncconsumer( + kafka_servers: KafkaServers, + loop: asyncio.AbstractEventLoop, +) -> AsyncGenerator[AsyncKafkaConsumer, None]: + asyncconsumer = AsyncKafkaConsumer( + bootstrap_servers=kafka_servers.bootstrap_servers, + loop=loop, + auto_offset_reset="earliest", + enable_auto_commit=False, + topic_metadata_refresh_interval_ms=200, # Speed things up for consumer tests to discover topics, etc. + ) + await asyncconsumer.start() + yield asyncconsumer + await asyncconsumer.stop() + + +@pytest.fixture(scope="function", name="registry_cluster") +async def fixture_registry_cluster( + karapace_container: KarapaceContainer, + loop: asyncio.AbstractEventLoop, # pylint: disable=unused-argument +) -> RegistryDescription: + protocol = "http" + endpoint = RegistryEndpoint( + protocol, karapace_container.config().registry_host, karapace_container.config().registry_port + ) + return RegistryDescription(endpoint, karapace_container.config().topic_name) + + +@pytest.fixture(scope="function", name="registry_async_client") +async def fixture_registry_async_client( + request: SubRequest, + basic_auth: BasicAuth, + registry_cluster: RegistryDescription, + loop: asyncio.AbstractEventLoop, # pylint: disable=unused-argument +) -> AsyncGenerator[Client, None]: + client = Client( + server_uri=registry_cluster.endpoint.to_url(), + server_ca=request.config.getoption("server_ca"), + session_auth=basic_auth, + ) + try: + yield client + finally: + await client.close() + + +@pytest.fixture(scope="function", name="new_topic") +def topic_fixture(admin_client: KafkaAdminClient) -> NewTopic: + topic_name = secrets.token_hex(4) + return admin_client.new_topic(topic_name, num_partitions=1, replication_factor=1) diff --git a/src/karapace/routers/__init__.py b/tests/e2e/instrumentation/__init__.py similarity index 100% rename from src/karapace/routers/__init__.py rename to tests/e2e/instrumentation/__init__.py diff --git a/tests/integration/instrumentation/test_prometheus.py b/tests/e2e/instrumentation/test_prometheus.py similarity index 100% rename from tests/integration/instrumentation/test_prometheus.py rename to tests/e2e/instrumentation/test_prometheus.py diff --git a/tests/integration/instrumentation/__init__.py b/tests/e2e/kafka/__init__.py similarity index 100% rename from tests/integration/instrumentation/__init__.py rename to tests/e2e/kafka/__init__.py diff --git a/tests/integration/kafka/test_admin.py b/tests/e2e/kafka/test_admin.py similarity index 100% rename from tests/integration/kafka/test_admin.py rename to tests/e2e/kafka/test_admin.py diff --git a/tests/integration/kafka/test_consumer.py b/tests/e2e/kafka/test_consumer.py similarity index 100% rename from tests/integration/kafka/test_consumer.py rename to tests/e2e/kafka/test_consumer.py diff --git a/tests/integration/kafka/test_producer.py b/tests/e2e/kafka/test_producer.py similarity index 100% rename from tests/integration/kafka/test_producer.py rename to tests/e2e/kafka/test_producer.py diff --git a/tests/integration/kafka/__init__.py b/tests/e2e/schema_registry/__init__.py similarity index 100% rename from tests/integration/kafka/__init__.py rename to tests/e2e/schema_registry/__init__.py diff --git a/tests/integration/schema_registry/test_jsonschema.py b/tests/e2e/schema_registry/test_jsonschema.py similarity index 100% rename from tests/integration/schema_registry/test_jsonschema.py rename to tests/e2e/schema_registry/test_jsonschema.py diff --git a/tests/integration/test_karapace.py b/tests/e2e/test_karapace.py similarity index 100% rename from tests/integration/test_karapace.py rename to tests/e2e/test_karapace.py diff --git a/tests/integration/schema_registry/__init__.py b/tests/integration/schema_registry/__init__.py deleted file mode 100644 index e69de29bb..000000000