docker-compose.yml

# docker-compose.yml
services:
  kafka:
    restart: always
    image: bitnami/kafka
    ports:
      - "9093:9093"
    volumes:
      - "./server/kafka-data:/bitnami"
    environment:
      - KAFKA_ENABLE_KRAFT=yes
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9093,CONTROLLER://:2181
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9093
      # - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9093
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_KRAFT_CLUSTER_ID=MkU3OEVBNTcwNTJENDM2Qk
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=false
  app:
    build:
      context: .
      dockerfile: Dockerfile.app
    ports:
      - "30001:8000"
    depends_on:
      redis:
        condition: service_healthy
    environment:
      - REDIS_URL=redis://redis:6379/0
      - KAFKA_BOOTSTRAP_SERVERS=kafka:9093
      - LOG_LEVEL=DEBUG
    command:
      ["poetry", "run", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
  worker:
    build:
      context: .
      dockerfile: Dockerfile.app
    depends_on:
      redis:
        condition: service_healthy
    environment:
      - REDIS_URL=redis://redis:6379/0
      - MODULE_NAME=stream_inference.app
      - KAFKA_BOOTSTRAP_SERVERS=kafka:9093
      - LOG_LEVEL=DEBUG
      - C_FORCE_ROOT=true # needed when using pickle serializer in celery + running as root - remove when we dont run as root
    command:
      'sh -c "cd tasks && poetry run celery -A $$MODULE_NAME worker --loglevel=info"'
  redis:
    image: redis:alpine
    ports:
      - "6379:6379"
    healthcheck:
      test: [ "CMD", "redis-cli", "ping" ]