docker-compose.yml

version: '3.9'
services:
  trino:
    image: 'trinodb/trino:latest'
    hostname: trino
    ports:
      - '8080:8080'
    volumes:
      - ./conf/trino:/etc/trino:ro
      - ./data/trino:/data/trino:rw
      - ./log/trino:/log/trino:rw
      - ./data/stage:/stage:rw
    networks:
      - ndsnet
    depends_on:
      - metastore
      - minio

  spark:
    build: spark/
    hostname: spark
    container_name: spark
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    volumes:
      - ./data/spark/warehouse:/home/iceberg/warehouse
      - ./data/stage:/home/data/stage
      - ./data/spark/notebooks:/home/iceberg/notebooks/notebooks
    ports:
      - 8888:8888 # Spark notebook port
      - 8081:8081 # Spark master web ui port
      - 8082:8082 # Spark worker web ui port
      - 18080:18080 # Spark history web ui port
    networks:
      - ndsnet
    command: 
      - notebook
    depends_on:
      - metastore
      - minio

  mariadb:
    image: 'mariadb:latest'
    hostname: mariadb
    ports:
      - '3306:3306'
    volumes:
      - ./data/mariadb:/var/lib/mysql  
    environment:
      - MYSQL_ROOT_PASSWORD=admin
      - MYSQL_USER=admin
      - MYSQL_PASSWORD=admin
      - MYSQL_DATABASE=metastore_db
    networks:
      - ndsnet

  metastore:
    hostname: metastore
    build: metastore/
    ports:
      - '9083:9083' # Metastore Thrift
    volumes:
      - ./conf/metastore/metastore-site.xml:/opt/metastore/conf/metastore-site.xml:ro
    environment:
      - METASTORE_DB_HOSTNAME=mariadb
      - HADOOP_HOME=/opt/hadoop
      - HIVE_HOME=/opt/metastore
    networks:
      - ndsnet
    depends_on:
      - mariadb

  minio:
    hostname: minio
    image: 'minio/minio:latest'
    container_name: minio
    ports:
      - '9000:9000'
      - '9001:9001'
    volumes:
      - ./data/minio:/data
    environment:
      - MINIO_ROOT_USER=minio
      - MINIO_ROOT_PASSWORD=minio123
    command: server /data --console-address ":9001"
    networks:
      - ndsnet

  mc:
    depends_on:
      - minio
    image: minio/mc
    container_name: mc
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    networks:
      - ndsnet
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 minio minio123) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc rm -r --force minio/test;
      /usr/bin/mc mb minio/test;
      /usr/bin/mc policy set public minio/test;
      exit 0;
      "      

networks:
  ndsnet:
    driver: bridge