From e9604ba96d66ffb2aab6fae11b03b50cf7675ecd Mon Sep 17 00:00:00 2001 From: rokh-conduktor <88239458+rokh-conduktor@users.noreply.github.com> Date: Wed, 27 Oct 2021 11:19:13 +0200 Subject: [PATCH] Use docker volumes to persist states (#96) * add docker volumes * update all recipes :recycle: * fix typo * add doc :pencil: --- README.md | 31 +++++++++++- ...ultiple-kafka-multiple-schema-registry.yml | 47 ++++++++++++++++-- zk-multiple-kafka-multiple.yml | 48 +++++++++++++++++-- zk-multiple-kafka-single.yml | 35 +++++++++++++- zk-single-kafka-multiple.yml | 27 ++++++++++- zk-single-kafka-single.yml | 17 ++++++- 6 files changed, 194 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c573b96..e0d08dc 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ docker-compose -f zk-single-kafka-single.yml up docker-compose -f zk-single-kafka-single.yml down ``` +Clean state (zookeeper & kafka) with : +``` +docker-compose -f zk-single-kafka-single.yml down +docker-compose -f zk-single-kafka-single.yml rm +``` + ## Single Zookeeper / Multiple Kafka If you want to have three brokers and experiment with kafka replication / fault-tolerance. @@ -76,6 +82,12 @@ docker-compose -f zk-single-kafka-multiple.yml up docker-compose -f zk-single-kafka-multiple.yml down ``` +Clean state (zookeeper & kafka) with : +``` +docker-compose -f zk-single-kafka-multiple.yml down +docker-compose -f zk-single-kafka-multiple.yml rm +``` + ## Multiple Zookeeper / Single Kafka If you want to have three zookeeper nodes and experiment with zookeeper fault-tolerance. @@ -90,6 +102,11 @@ docker-compose -f zk-multiple-kafka-single.yml up docker-compose -f zk-multiple-kafka-single.yml down ``` +Clean state (zookeeper & kafka) with : +``` +docker-compose -f zk-multiple-kafka-single.yml down +docker-compose -f zk-multiple-kafka-single.yml rm +``` ## Multiple Zookeeper / Multiple Kafka @@ -104,6 +121,12 @@ docker-compose -f zk-multiple-kafka-multiple.yml up docker-compose -f zk-multiple-kafka-multiple.yml down ``` +Clean state (zookeeper & kafka) with : +``` +docker-compose -f zk-multiple-kafka-multiple.yml down +docker-compose -f zk-multiple-kafka-multiple.yml rm +``` + ## Full stack Need a UI? We recommend using [Conduktor](https://conduktor.io) as your tool to bring a unified UI to all these components @@ -123,6 +146,12 @@ Need a UI? We recommend using [Conduktor](https://conduktor.io) as your tool to docker-compose -f full-stack.yml down ``` +Clean state (zookeeper & kafka) with : +``` +docker-compose -f full-stack.yml down +docker-compose -f full-stack.yml rm +``` + # FAQ ## Kafka @@ -133,7 +162,7 @@ A: Add the following line to your docker-compose environment variables: `KAFKA_L **Q: How do I delete data to start fresh?** -A: Your data is persisted from within the docker compose folder, so if you want for example to reset the data in the full-stack docker compose, do a `docker-compose -f full-stack.yml down`. +A: Your data is persisted from within docker volumes , so if you want for example to reset the data in the full-stack docker compose, do a `docker-compose -f full-stack.yml rm`. **Q: Can I change the zookeeper ports?** diff --git a/zk-multiple-kafka-multiple-schema-registry.yml b/zk-multiple-kafka-multiple-schema-registry.yml index 15c011d..05d3f5b 100644 --- a/zk-multiple-kafka-multiple-schema-registry.yml +++ b/zk-multiple-kafka-multiple-schema-registry.yml @@ -1,5 +1,14 @@ -version: '2.1' - +version: '3.8' +volumes: + zkmultiple-kmultiple-sr_kafka1-data: + zkmultiple-kmultiple-sr_kafka2-data: + zkmultiple-kmultiple-sr_kafka3-data: + zkmultiple-kmultiple-sr_zoo1-data: + zkmultiple-kmultiple-sr_zoo2-data: + zkmultiple-kmultiple-sr_zoo3-data: + zkmultiple-kmultiple-sr_zoo1-log: + zkmultiple-kmultiple-sr_zoo2-log: + zkmultiple-kmultiple-sr_zoo3-log: services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 @@ -11,6 +20,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 + volumes: + - type: volume + source: zkmultiple-kmultiple-sr_zoo1-data + target: /data + - type: volume + source: zkmultiple-kmultiple-sr_zoo1-log + target: /datalog zoo2: image: confluentinc/cp-zookeeper:6.2.1 @@ -22,6 +38,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2182 ZOOKEEPER_SERVER_ID: 2 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 + volumes: + - type: volume + source: zkmultiple-kmultiple-sr_zoo2-data + target: /data + - type: volume + source: zkmultiple-kmultiple-sr_zoo2-log + target: /datalog zoo3: image: confluentinc/cp-zookeeper:6.2.1 @@ -33,7 +56,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2183 ZOOKEEPER_SERVER_ID: 3 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 - + volumes: + - type: volume + source: zkmultiple-kmultiple-sr_zoo3-data + target: /data + - type: volume + source: zkmultiple-kmultiple-sr_zoo3-log + target: /datalog kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -47,6 +76,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: zkmultiple-kmultiple-sr_kafka1-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 @@ -64,6 +97,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: zkmultiple-kmultiple-sr_kafka2-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 @@ -81,6 +118,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: zkmultiple-kmultiple-sr_kafka3-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml index 1c5a2dd..57a4883 100644 --- a/zk-multiple-kafka-multiple.yml +++ b/zk-multiple-kafka-multiple.yml @@ -1,5 +1,14 @@ -version: '2.1' - +version: '3.8' +volumes: + zkmultiple-kmultiple_kafka1-data: + zkmultiple-kmultiple_kafka2-data: + zkmultiple-kmultiple_kafka3-data: + zkmultiple-kmultiple_zoo1-data: + zkmultiple-kmultiple_zoo2-data: + zkmultiple-kmultiple_zoo3-data: + zkmultiple-kmultiple_zoo1-log: + zkmultiple-kmultiple_zoo2-log: + zkmultiple-kmultiple_zoo3-log: services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 @@ -11,7 +20,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 - + volumes: + - type: volume + source: zkmultiple-kmultiple_zoo1-data + target: /data + - type: volume + source: zkmultiple-kmultiple_zoo1-log + target: /datalog zoo2: image: confluentinc/cp-zookeeper:6.2.1 hostname: zoo2 @@ -22,6 +37,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2182 ZOOKEEPER_SERVER_ID: 2 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 + volumes: + - type: volume + source: zkmultiple-kmultiple_zoo2-data + target: /data + - type: volume + source: zkmultiple-kmultiple_zoo2-log + target: /datalog zoo3: image: confluentinc/cp-zookeeper:6.2.1 @@ -33,7 +55,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2183 ZOOKEEPER_SERVER_ID: 3 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 - + volumes: + - type: volume + source: zkmultiple-kmultiple_zoo3-data + target: /data + - type: volume + source: zkmultiple-kmultiple_zoo3-log + target: /datalog kafka1: @@ -48,6 +76,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: zkmultiple-kmultiple_kafka1-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 @@ -65,6 +97,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: zkmultiple-kmultiple_kafka2-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 @@ -82,6 +118,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: zkmultiple-kmultiple_kafka3-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 diff --git a/zk-multiple-kafka-single.yml b/zk-multiple-kafka-single.yml index 05a3555..4e25cd5 100644 --- a/zk-multiple-kafka-single.yml +++ b/zk-multiple-kafka-single.yml @@ -1,4 +1,12 @@ -version: '2.1' +version: '3.8' +volumes: + zkmultiple-ksingle_kafka1-data: + zkmultiple-ksingle_zoo1-data: + zkmultiple-ksingle_zoo2-data: + zkmultiple-ksingle_zoo3-data: + zkmultiple-ksingle_zoo1-log: + zkmultiple-ksingle_zoo2-log: + zkmultiple-ksingle_zoo3-log: services: zoo1: @@ -11,6 +19,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 + volumes: + - type: volume + source: zkmultiple-ksingle_zoo1-data + target: /data + - type: volume + source: zkmultiple-ksingle_zoo1-log + target: /datalog zoo2: image: confluentinc/cp-zookeeper:6.2.1 @@ -22,6 +37,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2182 ZOOKEEPER_SERVER_ID: 2 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 + volumes: + - type: volume + source: zkmultiple-ksingle_zoo2-data + target: /data + - type: volume + source: zkmultiple-ksingle_zoo2-log + target: /datalog zoo3: image: confluentinc/cp-zookeeper:6.2.1 @@ -33,6 +55,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2183 ZOOKEEPER_SERVER_ID: 3 ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 + volumes: + - type: volume + source: zkmultiple-ksingle_zoo3-data + target: /data + - type: volume + source: zkmultiple-ksingle_zoo3-log + target: /datalog kafka1: @@ -53,6 +82,10 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_JMX_PORT: 9999 KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} + volumes: + - type: volume + source: zkmultiple-ksingle_kafka1-data + target: /var/lib/kafka/data depends_on: - zoo1 - zoo2 diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml index f4dcca9..13218b3 100644 --- a/zk-single-kafka-multiple.yml +++ b/zk-single-kafka-multiple.yml @@ -1,4 +1,10 @@ -version: '2.1' +version: '3.8' +volumes: + sksingle-kmultiple_kafka1-data: + sksingle-kmultiple_kafka2-data: + sksingle-kmultiple_kafka3-data: + sksingle-kmultiple_zoo1-data: + sksingle-kmultiple_zoo1-log: services: zoo1: @@ -11,6 +17,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 + volumes: + - type: volume + source: sksingle-kmultiple_zoo1-data + target: /data + - type: volume + source: sksingle-kmultiple_zoo1-log + target: /datalog kafka1: @@ -25,6 +38,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: sksingle-kmultiple_kafka1-data + target: /var/lib/kafka/data depends_on: - zoo1 @@ -40,6 +57,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: sksingle-kmultiple_kafka2-data + target: /var/lib/kafka/data depends_on: - zoo1 @@ -56,5 +77,9 @@ services: KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + volumes: + - type: volume + source: sksingle-kmultiple_kafka3-data + target: /var/lib/kafka/data depends_on: - zoo1 diff --git a/zk-single-kafka-single.yml b/zk-single-kafka-single.yml index b118767..5bd9350 100644 --- a/zk-single-kafka-single.yml +++ b/zk-single-kafka-single.yml @@ -1,4 +1,8 @@ -version: '2.1' +version: '3.8' +volumes: + zksingle-ksingle_kafka1-data: + zksingle-ksingle_zoo1-data: + zksingle-ksingle_zoo1-log: services: zoo1: @@ -11,6 +15,13 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 + volumes: + - type: volume + source: zksingle-ksingle_zoo1-data + target: /data + - type: volume + source: zksingle-ksingle_zoo1-log + target: /datalog kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -31,5 +42,9 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_JMX_PORT: 9999 KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} + volumes: + - type: volume + source: zksingle-ksingle_kafka1-data + target: /var/lib/kafka/data depends_on: - zoo1 \ No newline at end of file