Use docker volumes to persist states (#96)

* add docker volumes

* update all recipes ♻️

* fix typo

* add doc 📝

README.md

@@ -62,6 +62,12 @@ docker-compose -f zk-single-kafka-single.yml up
 docker-compose -f zk-single-kafka-single.yml down
 ```
 
+Clean state (zookeeper & kafka) with : 
+```
+docker-compose -f zk-single-kafka-single.yml down
+docker-compose -f zk-single-kafka-single.yml rm
+```
+
 ## Single Zookeeper / Multiple Kafka
 
 If you want to have three brokers and experiment with kafka replication / fault-tolerance.
@@ -76,6 +82,12 @@ docker-compose -f zk-single-kafka-multiple.yml up
 docker-compose -f zk-single-kafka-multiple.yml down
 ```
 
+Clean state (zookeeper & kafka) with :
+```
+docker-compose -f zk-single-kafka-multiple.yml down
+docker-compose -f zk-single-kafka-multiple.yml rm
+```
+
 ## Multiple Zookeeper / Single Kafka
 
 If you want to have three zookeeper nodes and experiment with zookeeper fault-tolerance.
@@ -90,6 +102,11 @@ docker-compose -f zk-multiple-kafka-single.yml up
 docker-compose -f zk-multiple-kafka-single.yml down
 ```
 
+Clean state (zookeeper & kafka) with :
+```
+docker-compose -f zk-multiple-kafka-single.yml down
+docker-compose -f zk-multiple-kafka-single.yml rm
+```
 
 ## Multiple Zookeeper / Multiple Kafka
 
@@ -104,6 +121,12 @@ docker-compose -f zk-multiple-kafka-multiple.yml up
 docker-compose -f zk-multiple-kafka-multiple.yml down
 ```
 
+Clean state (zookeeper & kafka) with :
+```
+docker-compose -f zk-multiple-kafka-multiple.yml down
+docker-compose -f zk-multiple-kafka-multiple.yml rm
+```
+
 ## Full stack
 
 Need a UI? We recommend using [Conduktor](https://conduktor.io) as your tool to bring a unified UI to all these components
@@ -123,6 +146,12 @@ Need a UI? We recommend using [Conduktor](https://conduktor.io) as your tool to
  docker-compose -f full-stack.yml down
  ```
 
+Clean state (zookeeper & kafka) with :
+```
+docker-compose -f full-stack.yml down
+docker-compose -f full-stack.yml rm
+```
+
 # FAQ
 
 ## Kafka
@@ -133,7 +162,7 @@ A: Add the following line to your docker-compose environment variables: `KAFKA_L
 
 **Q: How do I delete data to start fresh?**
 
-A: Your data is persisted from within the docker compose folder, so if you want for example to reset the data in the full-stack docker compose, do a `docker-compose -f full-stack.yml down`.
+A: Your data is persisted from within docker volumes , so if you want for example to reset the data in the full-stack docker compose, do a `docker-compose -f full-stack.yml rm`.
 
 **Q: Can I change the zookeeper ports?**
 

zk-multiple-kafka-multiple-schema-registry.yml

@@ -1,5 +1,14 @@
-version: '2.1'
-
+version: '3.8'
+volumes:
+  zkmultiple-kmultiple-sr_kafka1-data:
+  zkmultiple-kmultiple-sr_kafka2-data:
+  zkmultiple-kmultiple-sr_kafka3-data:
+  zkmultiple-kmultiple-sr_zoo1-data:
+  zkmultiple-kmultiple-sr_zoo2-data:
+  zkmultiple-kmultiple-sr_zoo3-data:
+  zkmultiple-kmultiple-sr_zoo1-log:
+  zkmultiple-kmultiple-sr_zoo2-log:
+  zkmultiple-kmultiple-sr_zoo3-log:
 services:
   zoo1:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -11,6 +20,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple-sr_zoo1-data
+        target: /data
+      - type: volume
+        source: zkmultiple-kmultiple-sr_zoo1-log
+        target: /datalog
 
   zoo2:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -22,6 +38,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2182
       ZOOKEEPER_SERVER_ID: 2
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple-sr_zoo2-data
+        target: /data
+      - type: volume
+        source: zkmultiple-kmultiple-sr_zoo2-log
+        target: /datalog
 
   zoo3:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -33,7 +56,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2183
       ZOOKEEPER_SERVER_ID: 3
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
-
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple-sr_zoo3-data
+        target: /data
+      - type: volume
+        source: zkmultiple-kmultiple-sr_zoo3-log
+        target: /datalog
 
   kafka1:
     image: confluentinc/cp-kafka:6.2.1
@@ -47,6 +76,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple-sr_kafka1-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2
@@ -64,6 +97,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 2
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple-sr_kafka2-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2
@@ -81,6 +118,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 3
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple-sr_kafka3-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2

zk-multiple-kafka-multiple.yml

@@ -1,5 +1,14 @@
-version: '2.1'
-
+version: '3.8'
+volumes:
+  zkmultiple-kmultiple_kafka1-data:
+  zkmultiple-kmultiple_kafka2-data:
+  zkmultiple-kmultiple_kafka3-data:
+  zkmultiple-kmultiple_zoo1-data:
+  zkmultiple-kmultiple_zoo2-data:
+  zkmultiple-kmultiple_zoo3-data:
+  zkmultiple-kmultiple_zoo1-log:
+  zkmultiple-kmultiple_zoo2-log:
+  zkmultiple-kmultiple_zoo3-log:
 services:
   zoo1:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -11,7 +20,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
-
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple_zoo1-data
+        target: /data
+      - type: volume
+        source: zkmultiple-kmultiple_zoo1-log
+        target: /datalog
   zoo2:
     image: confluentinc/cp-zookeeper:6.2.1
     hostname: zoo2
@@ -22,6 +37,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2182
       ZOOKEEPER_SERVER_ID: 2
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple_zoo2-data
+        target: /data
+      - type: volume
+        source: zkmultiple-kmultiple_zoo2-log
+        target: /datalog
 
   zoo3:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -33,7 +55,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2183
       ZOOKEEPER_SERVER_ID: 3
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
-
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple_zoo3-data
+        target: /data
+      - type: volume
+        source: zkmultiple-kmultiple_zoo3-log
+        target: /datalog
 
 
   kafka1:
@@ -48,6 +76,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple_kafka1-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2
@@ -65,6 +97,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 2
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple_kafka2-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2
@@ -82,6 +118,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 3
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: zkmultiple-kmultiple_kafka3-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2

zk-multiple-kafka-single.yml

@@ -1,4 +1,12 @@
-version: '2.1'
+version: '3.8'
+volumes:
+  zkmultiple-ksingle_kafka1-data:
+  zkmultiple-ksingle_zoo1-data:
+  zkmultiple-ksingle_zoo2-data:
+  zkmultiple-ksingle_zoo3-data:
+  zkmultiple-ksingle_zoo1-log:
+  zkmultiple-ksingle_zoo2-log:
+  zkmultiple-ksingle_zoo3-log:
 
 services:
   zoo1:
@@ -11,6 +19,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
+    volumes:
+      - type: volume
+        source: zkmultiple-ksingle_zoo1-data
+        target: /data
+      - type: volume
+        source: zkmultiple-ksingle_zoo1-log
+        target: /datalog
 
   zoo2:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -22,6 +37,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2182
       ZOOKEEPER_SERVER_ID: 2
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
+    volumes:
+      - type: volume
+        source: zkmultiple-ksingle_zoo2-data
+        target: /data
+      - type: volume
+        source: zkmultiple-ksingle_zoo2-log
+        target: /datalog
 
   zoo3:
     image: confluentinc/cp-zookeeper:6.2.1
@@ -33,6 +55,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2183
       ZOOKEEPER_SERVER_ID: 3
       ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
+    volumes:
+      - type: volume
+        source: zkmultiple-ksingle_zoo3-data
+        target: /data
+      - type: volume
+        source: zkmultiple-ksingle_zoo3-log
+        target: /datalog
 
 
   kafka1:
@@ -53,6 +82,10 @@ services:
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_JMX_PORT: 9999
       KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
+    volumes:
+      - type: volume
+        source: zkmultiple-ksingle_kafka1-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2

zk-single-kafka-multiple.yml

@@ -1,4 +1,10 @@
-version: '2.1'
+version: '3.8'
+volumes:
+  sksingle-kmultiple_kafka1-data:
+  sksingle-kmultiple_kafka2-data:
+  sksingle-kmultiple_kafka3-data:
+  sksingle-kmultiple_zoo1-data:
+  sksingle-kmultiple_zoo1-log:
 
 services:
   zoo1:
@@ -11,6 +17,13 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_SERVERS: zoo1:2888:3888
+    volumes:
+      - type: volume
+        source: sksingle-kmultiple_zoo1-data
+        target: /data
+      - type: volume
+        source: sksingle-kmultiple_zoo1-log
+        target: /datalog
 
 
   kafka1:
@@ -25,6 +38,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: sksingle-kmultiple_kafka1-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
 
@@ -40,6 +57,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 2
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: sksingle-kmultiple_kafka2-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1
 
@@ -56,5 +77,9 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 3
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - type: volume
+        source: sksingle-kmultiple_kafka3-data
+        target: /var/lib/kafka/data
     depends_on:
       - zoo1