docker-compose.yml

version: '3.8'

services:
  # 1. Zookeeper 
  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - twitter-network

  # 2. Kafka Broker
  kafka:
    image: confluentinc/cp-kafka:7.5.0
    hostname: kafka
    container_name: kafka
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    depends_on:
      - zookeeper
    networks:
      - twitter-network
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:9092"]
      interval: 10s
      timeout: 10s
      retries: 5

  # 3. HDFS Namenode 
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    platform: linux/amd64
    container_name: namenode
    hostname: namenode
    ports:
      - "9870:9870"
      - "9000:9000"
    environment:
      - CLUSTER_NAME=test
      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
      - CORE_CONF_hadoop_http_staticuser_user=root
      - HDFS_CONF_dfs_webhdfs_enabled=true
      - HDFS_CONF_dfs_permissions_enabled=false
      - HDFS_CONF_dfs_replication=1
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
      - ./data:/data
    networks:
      - twitter-network

  # 4. HDFS Datanode
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    platform: linux/amd64
    container_name: datanode
    hostname: datanode
    ports:
      - "9864:9864"
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
      - CORE_CONF_hadoop_http_staticuser_user=root
      - HDFS_CONF_dfs_webhdfs_enabled=true
      - HDFS_CONF_dfs_permissions_enabled=false
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    depends_on:
      - namenode
    networks:
      - twitter-network

  # 5. PostgreSQL Hive Metastore
  postgres:
    image: postgres:14
    container_name: postgres
    hostname: postgres
    ports:
      - "5432:5432"
    environment:
      POSTGRES_DB: metastore
      POSTGRES_USER: hive
      POSTGRES_PASSWORD: hive
    volumes:
      - postgres_data:/var/lib/postgresql/data
    networks:
      - twitter-network
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U hive -d metastore -h localhost"]
      interval: 10s
      timeout: 5s
      retries: 10

  # 6. Hive Metastore
  hive-metastore:
    image: apache/hive:4.0.0
    container_name: hive-metastore
    hostname: hive-metastore
    ports:
      - "9083:9083"
    environment:
      SERVICE_NAME: metastore
      DB_DRIVER: postgres
      SERVICE_OPTS: "-Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore -Djavax.jdo.option.ConnectionUserName=hive -Djavax.jdo.option.ConnectionPassword=hive"
      HIVE_AUX_JARS_PATH: /opt/hive/lib/postgresql-42.7.4.jar
    depends_on:
      postgres:
        condition: service_healthy
      namenode:
        condition: service_started
    volumes:
      - ./drivers/postgresql-42.7.4.jar:/opt/hive/lib/postgresql-42.7.4.jar:ro
    networks:
      - twitter-network

  # 7. Hive Server
  hive-server:
    image: apache/hive:4.0.0
    container_name: hive-server
    hostname: hive-server
    ports:
      - "10000:10000"
      - "10002:10002"
    environment:
      SERVICE_NAME: hiveserver2
      SERVICE_OPTS: "-Dhive.metastore.uris=thrift://hive-metastore:9083"
      IS_RESUME: "true"
      HIVE_AUX_JARS_PATH: /opt/hive/lib/postgresql-42.7.4.jar
    depends_on:
      hive-metastore:
        condition: service_started
      namenode:
        condition: service_started
    volumes:
      - ./drivers/postgresql-42.7.4.jar:/opt/hive/lib/postgresql-42.7.4.jar:ro
    networks:
      - twitter-network

  # 8. Spark Master
  spark-master:
    image: apache/spark:3.5.3
    container_name: spark-master
    hostname: spark-master
    ports:
      - "7077:7077"
      - "8080:8080"
    volumes:
      - ./data:/data
    depends_on:
      - namenode
    networks:
      - twitter-network
    command: /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master

  # 9. Spark Worker
  spark-worker:
    image: apache/spark:3.5.3
    container_name: spark-worker
    hostname: spark-worker
    ports:
      - "8081:8081"
    volumes:
      - ./data:/data
    depends_on:
      - spark-master
    networks:
      - twitter-network
    command: /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077

  # 10. Flink Job Manager
  jobmanager:
    image: flink:1.17.0-scala_2.12-java11
    container_name: jobmanager
    hostname: jobmanager
    ports:
      - "8082:8081"
      - "6123:6123"
    command: jobmanager
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager
    volumes:
      - ./flink-jobs:/opt/flink/usrlib
      - ./data:/data
    networks:
      - twitter-network

  # 11. Flink Task Manager
  taskmanager:
    image: flink:1.17.0-scala_2.12-java11
    container_name: taskmanager
    hostname: taskmanager
    depends_on:
      - jobmanager
    command: taskmanager
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager
    volumes:
      - ./flink-jobs:/opt/flink/usrlib
      - ./data:/data
    networks:
      - twitter-network

volumes:
  hadoop_namenode:
  hadoop_datanode:
  postgres_data:

networks:
  twitter-network:
    driver: bridge