docker-compose.yml

services:
    mongodb:
        image: mongo:latest
        container_name: mongodb_nyt
        volumes:
          - ./docker_mongo:/data/db
          - ./data:/data/db/data
          - .env:/data/db/.env
          - ./mongodb/init_db.sh:/docker-entrypoint-initdb.d/init_db.sh:ro

        ports:
          - 27017:27017
        env_file:
          - .env
        environment:
          - MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME}
          - MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD}
          - MONGO_INITDB_DATABASE=${MONGO_INITDB_DATABASE}
        networks:
          - mongo-compose-network

    mongoexpress:
        container_name: mongo-express
        image: mongo-express:1.0.0-20
        restart: on-failure
        env_file:
          - .env
        depends_on:
          - mongodb
        networks:
          - mongo-compose-network
        environment:
          - ME_CONFIG_MONGODB_SERVER=mongodb
          - ME_CONFIG_MONGODB_ADMINUSERNAME=${MONGO_INITDB_ROOT_USERNAME}
          - ME_CONFIG_MONGODB_ADMINPASSWORD=${MONGO_INITDB_ROOT_PASSWORD}
          - ME_CONFIG_MONGODB_ENABLE_ADMIN=true
          - ME_CONFIG_BASICAUTH_USERNAME=${MONGOEXPRESS_LOGIN}
          - ME_CONFIG_BASICAUTH_PASSWORD=${MONGOEXPRESS_PASSWORD}
        ports:
          - '8082:8081'
        volumes:
          - ./docker_mongo:/data/db

    zookeeper:
      container_name: zookeeper
      image: wurstmeister/zookeeper
      ports:
        - "2181"
      networks:
        - kafka
      healthcheck:
        test: nc -z localhost 2181 || exit -1
        interval: 45s
        timeout: 10s
        retries: 5

    kafka1:
      image: wurstmeister/kafka
      restart: on-failure
      hostname: kafka1
      container_name: kafka1
      depends_on:
        - zookeeper
      ports:
        - "9092:9092"
      environment:
        KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:29092,EXTERNAL://localhost:9092
        KAFKA_LISTENERS: INTERNAL://kafka1:29092,EXTERNAL://0.0.0.0:9092
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
        KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
        KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
        KAFKA_BROKER_ID: 1
      healthcheck:
        test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "localhost:9092"]
        interval: 45s
        timeout: 10s
        retries: 5
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
      networks:
        - kafka

    kafka2:
      image: wurstmeister/kafka
      restart: on-failure
      hostname: kafka2
      container_name: kafka2
      depends_on:
        - zookeeper
      ports:
        - "9093:9093"
      environment:
        KAFKA_BROKER_ID: 2
        KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
        KAFKA_LISTENERS: INTERNAL://kafka2:29093,EXTERNAL://0.0.0.0:9093
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
        KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
        KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:29093,EXTERNAL://localhost:9093
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
      networks:
        - kafka

    kafka3:
      image: wurstmeister/kafka
      restart: on-failure
      hostname: kafka3
      container_name: kafka3
      depends_on:
        - zookeeper
      ports:
        - "9094:9094"
      environment:
        KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:29094,EXTERNAL://localhost:9094
        KAFKA_LISTENERS: INTERNAL://kafka3:29094,EXTERNAL://0.0.0.0:9094
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
        KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
        KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
        KAFKA_BROKER_ID: 3
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
      networks:
        - kafka

    kafkaui:
      image: provectuslabs/kafka-ui
      depends_on:
        - kafka1
        - zookeeper
      ports:
        - "8089:8080"
      environment:
        KAFKA_CLUSTERS_0_NAME: kafka1
        KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka1:29092"
        KAFKA_CLUSTERS_1_NAME: kafka2
        KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS: "kafka2:29093"
        KAFKA_CLUSTERS_2_NAME: kafka3
        KAFKA_CLUSTERS_2_BOOTSTRAPSERVERS: "kafka3:29094"
      networks:
        - kafka

    app:
      container_name: app
      restart: on-failure
      build:
        context: ./app
        dockerfile: Dockerfile
#      image: registry.gitlab.com/nyt_news4/nyt_news/app:latest
#      image: docker.io/linenlp/app:latest
      env_file:
        - .env
      depends_on:
        kafka1:
          condition: service_healthy
        prediction:
          condition: service_healthy
      command: >
        bash -c "python nyt_news/main.py"
      networks:
        - mongo-compose-network
        - kafka

    consumer:
      container_name: consumer
      restart: on-failure
      build:
        context: ./consumer
        dockerfile: Dockerfile
      env_file:
        - .env
#      image: registry.gitlab.com/nyt_news4/nyt_news/consumer:latest
#      image: docker.io/linenlp/consumer:latest
      depends_on:
        kafka1:
          condition: service_healthy
      command: python consumer.py
      networks:
        - kafka
      ports:
        - "8001:8000"

    polarity:
      container_name: polarity_trainer
      build:
        context: ./ml/polarity
        dockerfile: Dockerfile
      env_file:
        - .env
#      image: registry.gitlab.com/nyt_news4/nyt_news/polarity:latest
#      image: docker.io/linenlp/polarity:latest
      profiles: [build]
      networks:
        - mongo-compose-network
      volumes:
        - ./ml/models:/models

    prediction:
      container_name: predict
      build:
        context: ./ml/predict
        dockerfile: Dockerfile
      env_file:
        - .env
#      image: registry.gitlab.com/nyt_news4/nyt_news/prediction:latest
#      image: docker.io/linenlp/prediction:latest
      networks:
        - mongo-compose-network
      ports:
        - '8005:8005'
      volumes:
        - ./ml/models:/models
      healthcheck:
        test: curl --fail http://localhost:8005/health || exit 1
        interval: 45s
        timeout: 10s
        retries: 5

    archive:
      container_name: archive
      restart: on-failure
      build:
        context: ./nyt_api_archive
        dockerfile: Dockerfile
#      profiles: [build]
      env_file:
        - .env
#      image: registry.gitlab.com/nyt_news4/nyt_news/prediction:latest
#      image: docker.io/linenlp/prediction:latest
      command: >
        bash -c "python main.py"
      depends_on:
        prediction:
          condition: service_healthy
      networks:
        - mongo-compose-network

networks:
    kafka:
    mongo-compose-network: