-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdocker-compose.yml
executable file
·108 lines (98 loc) · 2.98 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
version: "3"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
volumes:
- namenode:/hadoop/dfs/name
environment:
- CLUSTER_NAME=test
env_file:
- ./hadoop-hive.env
ports:
- "50070:50070"
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes:
- datanode:/hadoop/dfs/data
env_file:
- ./hadoop-hive.env
environment:
SERVICE_PRECONDITION: "namenode:50070"
ports:
- "50075:50075"
resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop2.7.4-java8
environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075"
env_file:
- ./hadoop-hive.env
hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore
env_file:
- ./hadoop-hive.env
environment:
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
SERVICE_PRECONDITION: "hive-metastore:9083"
ports:
- "10000:10000"
hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore
env_file:
- ./hadoop-hive.env
command: /opt/hive/bin/hive --service metastore
environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432 resourcemanager:8088"
ports:
- "9083:9083"
hive-metastore-postgresql:
image: bde2020/hive-metastore-postgresql:2.3.0
ports:
- "5432:5432"
huedb:
image: postgres:12.1-alpine
volumes:
- pg_data:/var/lib/postgresl/data/
ports:
- "5432"
env_file:
- ./hadoop-hive.env
environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432 resourcemanager:8088 hive-metastore:9083"
hue:
image: gethue/hue:4.6.0
environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432 resourcemanager:8088 hive-metastore:9083 huedb:5000"
ports:
- "8888:8888"
volumes:
- ./hue-overrides.ini:/usr/share/hue/desktop/conf/hue-overrides.ini
links:
- huedb
pyspark-notebook:
image: jupyter/pyspark-notebook:latest #jupyter/datascience-notebook
container_name: pyspark-notebook
ports:
- "8889:8888"
- "8000:8000"
# volumes:
#- /home/ozi/pri:/home/jovyan
# - ./charts:/home/jovyan/work
# - ./imported_data:/home/jovyan/work/imported_data
# - ./processed_data:/home/jovyan/work/processed_data
# - ./raw_data:/home/jovyan/work/raw_data
environment:
- PYSPARK_PYTHON=python3
# Configuraciones de conexión a Hive
- HIVE_SERVER2_HOST=hive-server
- HIVE_SERVER2_PORT=10000
- HIVE_METASTORE_HOST=hive-metastore
- HIVE_METASTORE_PORT=9083
# Configuración de JDBC para Hive
- HIVE_JDBC_URL=jdbc:hive2://hive-server:10000/default
- HIVE_JDBC_DRIVER=org.apache.hive.jdbc.HiveDriver
- HIVE_JDBC_USER=hive
- HIVE_JDBC_PASSWORD=hive
volumes:
namenode:
datanode:
pg_data: