From d0e8ddd5a2f2edc15cfae9429b52cfcf6626a95a Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 22:02:17 +0200 Subject: [PATCH 1/8] refactor: setup Kafka cluster with KRaft Setup KRaft based Kafka cluster which does no rely on ZooKeeper therefore we can simplify the deployment a bit. --- docker-compose.yaml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index e5e983842..050622646 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,31 +1,30 @@ services: - zookeeper: - image: confluentinc/cp-zookeeper:7.4.0 - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - kafka: - image: confluentinc/cp-kafka:7.4.0 - depends_on: - - zookeeper + image: confluentinc/cp-kafka:7.7.1 ports: - - 127.0.0.1:29092:29092 + - "127.0.0.1:29092:29092" environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + CLUSTER_ID: ca497efe-9f82-4b84-890b-d9969a9a2e1c + KAFKA_BROKER_ID: 0 + KAFKA_PROCESS_ROLES: "broker,controller" + KAFKA_CONTROLLER_QUORUM_VOTERS: "0@kafka:9093" + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://localhost:29092 + KAFKA_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://0.0.0.0:29092,CONTROLLER://kafka:9093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + JMX_PORT: 9997 + KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=9997" healthcheck: - test: kafka-topics --bootstrap-server kafka:9092 --list + test: ["CMD-SHELL", "JMX_PORT=9998", "kafka-topics", "--bootstrap-server", "kafka:9092", "--list"] interval: 5s timeout: 3s - retries: 100 + retries: 30 clickhouse: image: clickhouse/clickhouse-server:23.3.9.55-alpine From a51c1c8bf1f6c42fe3857813a72a549d9d33993c Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 22:03:15 +0200 Subject: [PATCH 2/8] refactor: replace Redpanda Console with Kafka UI The Kafka UI has very similar feature set than Redpanda Console while certian functions are available without commercial lincese like: * edit/delete consumer groups * edit/recreate topics --- docker-compose.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 050622646..10e8f79e3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,16 +58,24 @@ services: # Development # docker compose --profile dev up - redpanda: - image: docker.redpanda.com/redpandadata/console:v2.3.1 + kafka-ui: + image: ghcr.io/kafbat/kafka-ui:main profiles: - dev depends_on: - kafka ports: - - 127.0.0.1:8084:8080 + - "127.0.0.1:8084:8080" environment: - KAFKA_BROKERS: kafka:9092 + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 + KAFKA_CLUSTERS_0_METRICS_PORT: 9997 + DYNAMIC_CONFIG_ENABLED: "true" + healthcheck: + test: ["CMD", "wget", "--spider", "http://kafka-ui:8080/"] + interval: 10s + timeout: 5s + retries: 30 wiretap: image: pb33f/wiretap:v0.0.49 From bd94eebad9c10ce762de46fa3b231d6cb8970c9c Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 22:17:31 +0200 Subject: [PATCH 3/8] chore: use alpine based image for Postgres server --- docker-compose.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 10e8f79e3..0bdaa53d0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -93,21 +93,23 @@ services: # Credit # docker compose --profile postgres up postgres: - image: postgres:14.9 + image: postgres:14.9-alpine profiles: - postgres ports: - - "5432:5432" + - "127.0.0.1:5432:5432" environment: - POSTGRES_USER=postgres - POSTGRES_DB=postgres - POSTGRES_PASSWORD=postgres volumes: - ./etc/postgres-init:/docker-entrypoint-initdb.d/ - command: - - "postgres" - - "-c" - - "wal_level=logical" + command: ["postgres", "-c", "wal_level=logical"] + healthcheck: + test: ["CMD-SHELL", "pg_isready", "-d", "$${POSTGRES_DB}", "-U", "$${POSTGRES_USER}"] + interval: 10s + timeout: 5s + retries: 30 svix: image: svix/svix-server:v1.29 From 145464dc329dd7b3669a57ea6143666b63c510c1 Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 22:23:33 +0200 Subject: [PATCH 4/8] refactor: Redis service --- docker-compose.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0bdaa53d0..b70f2ee6a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -52,9 +52,15 @@ services: redis: profiles: - redis - image: redis:7.0.12-alpine + image: redis:7.0-alpine ports: - - 127.0.0.1:6379:6379 + - "127.0.0.1:6379:6379" + command: ["redis-server", "--maxmemory", "50mb", "--maxmemory-policy", "noeviction", "--maxmemory-samples", "10"] + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + interval: 5s + timeout: 3s + retries: 30 # Development # docker compose --profile dev up From c0b4d9619f8608ba8a27f2e31f118288e0347be5 Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 22:41:49 +0200 Subject: [PATCH 5/8] chore: bump Clickhouse server version to v24.9 --- docker-compose.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b70f2ee6a..9cd22054d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,11 +27,11 @@ services: retries: 30 clickhouse: - image: clickhouse/clickhouse-server:23.3.9.55-alpine + image: clickhouse/clickhouse-server:24.9-alpine ports: - - 127.0.0.1:8123:8123 - - 127.0.0.1:9000:9000 - - 127.0.0.1:9009:9009 + - "127.0.0.1:8123:8123" + - "127.0.0.1:9000:9000" + - "127.0.0.1:9009:9009" environment: CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: default @@ -42,10 +42,10 @@ services: soft: 262144 hard: 262144 healthcheck: - test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1 + test: ["CMD", "wget", "--spider", "http://clickhouse:8123/ping"] interval: 5s timeout: 3s - retries: 100 + retries: 30 # Deduplication # docker compose --profile redis up From 3987bb52382ab870a56b59d91daa79caf3c32217 Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 22:54:47 +0200 Subject: [PATCH 6/8] chore: bump Svix server version to v1.37 --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9cd22054d..e2b72286f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -118,7 +118,7 @@ services: retries: 30 svix: - image: svix/svix-server:v1.29 + image: svix/svix-server:v1.37 profiles: - webhook environment: @@ -130,7 +130,7 @@ services: SVIX_LOG_LEVEL: "info" SVIX_QUEUE_TYPE: "redis" ports: - - "8071:8071" + - "127.0.0.1:8071:8071" depends_on: - postgres - redis From c7a31b513e7bfbe54828d3026da40f043252d36e Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Thu, 3 Oct 2024 23:04:43 +0200 Subject: [PATCH 7/8] refactor: inline Svix database init SQL script --- docker-compose.yaml | 13 +++++++++++-- etc/postgres-init/svix.sql | 7 ------- 2 files changed, 11 insertions(+), 9 deletions(-) delete mode 100644 etc/postgres-init/svix.sql diff --git a/docker-compose.yaml b/docker-compose.yaml index e2b72286f..b28e2e91a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -108,8 +108,9 @@ services: - POSTGRES_USER=postgres - POSTGRES_DB=postgres - POSTGRES_PASSWORD=postgres - volumes: - - ./etc/postgres-init:/docker-entrypoint-initdb.d/ + configs: + - source: svix_sql + target: /docker-entrypoint-initdb.d/svix.sql command: ["postgres", "-c", "wal_level=logical"] healthcheck: test: ["CMD-SHELL", "pg_isready", "-d", "$${POSTGRES_DB}", "-U", "$${POSTGRES_USER}"] @@ -134,3 +135,11 @@ services: depends_on: - postgres - redis + +configs: + svix_sql: + content: | + CREATE USER svix WITH PASSWORD 'svix'; + CREATE DATABASE svix; + GRANT ALL PRIVILEGES ON DATABASE svix TO svix; + ALTER DATABASE svix OWNER TO svix; diff --git a/etc/postgres-init/svix.sql b/etc/postgres-init/svix.sql deleted file mode 100644 index f0993185f..000000000 --- a/etc/postgres-init/svix.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE USER svix WITH PASSWORD 'svix'; - -CREATE DATABASE svix; - -GRANT ALL PRIVILEGES ON DATABASE svix TO svix; - -ALTER DATABASE svix OWNER TO svix; \ No newline at end of file From 7c811fb7edd7f83ee7d77e9b45e386912ae93b97 Mon Sep 17 00:00:00 2001 From: Krisztian Gacsal Date: Fri, 4 Oct 2024 00:11:59 +0200 Subject: [PATCH 8/8] chore: update Qucikstart --- quickstart/config.yaml | 11 ++++- quickstart/docker-compose.ci.yaml | 2 - quickstart/docker-compose.yaml | 69 ++++++++++--------------------- 3 files changed, 30 insertions(+), 52 deletions(-) diff --git a/quickstart/config.yaml b/quickstart/config.yaml index 027dfc25b..5c2844cb9 100644 --- a/quickstart/config.yaml +++ b/quickstart/config.yaml @@ -1,6 +1,6 @@ ingest: kafka: - broker: kafka:29092 + broker: kafka:9092 aggregation: clickhouse: @@ -10,8 +10,15 @@ sink: minCommitCount: 1 namespaceRefetch: 1s kafka: - brokers: kafka:29092 + brokers: kafka:9092 brokerAddressFamily: v4 + dedupe: + enabled: true + driver: redis + config: + address: redis:6379 + database: 0 + expiration: 768h # 32d postgres: url: postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable diff --git a/quickstart/docker-compose.ci.yaml b/quickstart/docker-compose.ci.yaml index 084c17a8c..33c4e1497 100644 --- a/quickstart/docker-compose.ci.yaml +++ b/quickstart/docker-compose.ci.yaml @@ -1,5 +1,3 @@ -version: "3.9" - services: openmeter: image: "local" diff --git a/quickstart/docker-compose.yaml b/quickstart/docker-compose.yaml index 1d4e4c785..d365ed22f 100644 --- a/quickstart/docker-compose.yaml +++ b/quickstart/docker-compose.yaml @@ -1,5 +1,3 @@ -version: "3.9" - services: openmeter: image: ghcr.io/openmeterio/openmeter:latest @@ -14,7 +12,7 @@ services: postgres: condition: service_healthy ports: - - 127.0.0.1:8888:8888 + - "127.0.0.1:8888:8888" volumes: - ./config.yaml:/etc/openmeter/config.yaml healthcheck: @@ -35,70 +33,45 @@ services: condition: service_healthy openmeter: condition: service_healthy + redis: + condition: service_healthy ports: - - 127.0.0.1:10000:10000 + - "127.0.0.1:10000:10000" volumes: - ./config.yaml:/etc/openmeter/config.yaml - zookeeper: + kafka: extends: file: ../docker-compose.yaml - service: zookeeper - - # Duplicating kafka here due to https://github.com/docker/compose/issues/11544 - kafka: - image: confluentinc/cp-kafka:7.4.0 - depends_on: - - zookeeper - ports: - - 127.0.0.1:29092:29092 - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://kafka:29092 - healthcheck: - test: kafka-topics --bootstrap-server kafka:9092 --list - interval: 5s - timeout: 3s - retries: 100 + service: kafka clickhouse: extends: file: ../docker-compose.yaml service: clickhouse + + redis: + image: redis:7.0-alpine + ports: + - "127.0.0.1:6379:6379" + command: ["redis-server", "--maxmemory", "50mb", "--maxmemory-policy", "noeviction", "--maxmemory-samples", "10"] healthcheck: - test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1 + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 5s timeout: 3s - retries: 100 - - # Profile: dedupe-redis - redis: - extends: - file: ../docker-compose.yaml - service: redis + retries: 30 postgres: - image: postgres:14.9 + image: postgres:14.9-alpine ports: - - "5432:5432" + - "127.0.0.1:5432:5432" environment: - POSTGRES_USER=postgres - POSTGRES_DB=postgres - POSTGRES_PASSWORD=postgres - command: - - "postgres" - - "-c" - - "wal_level=logical" + command: ["postgres", "-c", "wal_level=logical"] healthcheck: - test: ["CMD-SHELL", "pg_isready", "-d", "db_prod"] - interval: 5s - timeout: 60s - retries: 5 + test: ["CMD-SHELL", "pg_isready", "-d", "$${POSTGRES_DB}", "-U", "$${POSTGRES_USER}"] + interval: 10s + timeout: 5s + retries: 30