docker-compose.yml

version: '3'

services:
  zookeeper-1:
    hostname: zookeeper1
    image: confluentinc/cp-zookeeper:7.3.3.arm64
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 12181
      ZOOKEEPER_DATA_DIR: /zookeeper/data
      ZOOKEEPER_SERVERS: zookeeper1:22888:23888;zookeeper2:32888:33888;zookeeper3:42888:43888
    ports:
      - "12181:12181"
      - "22888:22888"
      - "23888:23888"
    volumes:
      - ./zookeeper/data/1:/zookeeper/data
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  zookeeper-2:
    hostname: zookeeper2
    image: confluentinc/cp-zookeeper:7.3.3.arm64
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_CLIENT_PORT: 22181
      ZOOKEEPER_DATA_DIR: /zookeeper/data
      ZOOKEEPER_SERVERS: zookeeper1:22888:23888;zookeeper2:32888:33888;zookeeper3:42888:43888
    ports:
      - "22181:22181"
      - "32888:32888"
      - "33888:33888"
    volumes:
      - ./zookeeper/data/2:/zookeeper/data
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  zookeeper-3:
    hostname: zookeeper3
    image: confluentinc/cp-zookeeper:7.3.3.arm64
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_DATA_DIR: /zookeeper/data
      ZOOKEEPER_SERVERS: zookeeper1:22888:23888;zookeeper2:32888:33888;zookeeper3:42888:43888
    ports:
      - "32181:32181"
      - "42888:42888"
      - "43888:43888"
    volumes:
      - ./zookeeper/data/3:/zookeeper/data
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  kafka-1:
    image: confluentinc/cp-kafka:7.3.3.arm64
    hostname: kafka1
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:12181,zookeeper2:22181,zookeeper3:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:19092
      KAFKA_LOG_DIRS: /kafka
      KAFKA_LOG_RETENTION_MS: 86400000 # 1 day
    ports:
      - "19092:19092"
    volumes:
      - ./kafka/logs/1:/kafka
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  kafka-2:
    image: confluentinc/cp-kafka:7.3.3.arm64
    hostname: kafka2
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:12181,zookeeper2:22181,zookeeper3:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:29092
      KAFKA_LOG_DIRS: /kafka
      KAFKA_LOG_RETENTION_MS: 86400000 # 1 day
    ports:
      - "29092:29092"
    volumes:
      - ./kafka/logs/2:/kafka
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  kafka-3:
    image: confluentinc/cp-kafka:7.3.3.arm64
    hostname: kafka3
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:12181,zookeeper2:22181,zookeeper3:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:39092
      KAFKA_LOG_DIRS: /kafka
      KAFKA_LOG_RETENTION_MS: 86400000 # 1 day
    ports:
      - "39092:39092"
    volumes:
      - ./kafka/logs/3:/kafka
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  akhq:
    image: tchiotludo/akhq:latest
    hostname: akhq
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
          connections:
            kafka:
              properties:
                bootstrap.servers: kafka1:19092,kafka2:29092,kafka3:39092
    ports:
      - "8080:8080"

  spark-master:
    image: bitnami/spark:latest
    container_name: spark-master
    ports:
      - "8888:8080"
      - "7077:7077"
    environment:
      - SPARK_MODE=master
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MASTER_PORT=7077
    volumes:
      - ./spark-streaming:/spark-streaming
      - ./kafka:/kafka
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 1G

  spark-worker-1:
    image: bitnami/spark:latest
    container_name: spark-worker-1
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    depends_on:
      - spark-master
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  spark-worker-2:
    image: bitnami/spark:latest
    container_name: spark-worker-2
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    depends_on:
      - spark-master
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  spark-worker-3:
    image: bitnami/spark:latest
    container_name: spark-worker-3
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
    depends_on:
      - spark-master
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G

  grafana:
    image: grafana/grafana-oss:8.2.6
    container_name: grafana
    restart: unless-stopped
#    environment:
#      - GF_INSTALL_PLUGINS=hamedkarbasi93-kafka-datasource
#    platform: 'linux/amd64' # hamedkarbasi93-kafka-datasource 이 플러그인에서 arm64를 지원하지 않음
    ports:
      - '3000:3000'
    volumes:
      - ./visualization/grafana:/var/lib/grafana

  prometheus:
    image: prom/prometheus
    container_name: prometheus
    volumes:
      - ./visualization/prometheus/config:/etc/prometheus
      - ./visualization/prometheus/volume:/prometheus
    ports:
      - "9090:9090"
    command: # web.enalbe-lifecycle은 api 재시작없이 설정파일들을 reload 할 수 있게 해줌
      - '--web.enable-lifecycle'
      - '--config.file=/etc/prometheus/prometheus.yml'
    restart: unless-stopped

  kafka-exporter:
    image: danielqsj/kafka-exporter:latest
    container_name: kafka-exporter
    ports:
      - "9308:9308"
    command:
      - --kafka.server=kafka1:19092
      - --kafka.server=kafka2:29092
      - --kafka.server=kafka3:39092

  influxdb:
    image: influxdb:2.7-alpine
    container_name: influxdb
    ports:
      - "8086:8086"
    environment:
      DOCKER_INFLUXDB_INIT_MODE: "setup"
      DOCKER_INFLUXDB_INIT_USERNAME: "admin"
      DOCKER_INFLUXDB_INIT_PASSWORD: "12341234"
      DOCKER_INFLUXDB_INIT_ORG: "log-streaming"
      DOCKER_INFLUXDB_INIT_BUCKET: "dashboard"
      DOCKER_INFLUXDB_INIT_RETENTION: "1w"
      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: "sample-token"
    volumes:
      - ./influxdb-data:/var/lib/influxdb

  kafka-influxdb-consumer:
    container_name: kafka-influxdb-consumer
    build:
      context: kafka-influxdb-consumer
    volumes:
      - ./kafka-influxdb-consumer:/app
    command: go run main.go