forked from Marcel-Jan/docker-hadoop-spark
-
Notifications
You must be signed in to change notification settings - Fork 5
/
docker-compose.yml
162 lines (141 loc) · 3.86 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
version: "3"
services:
######## Hadoop configuration ########
namenode:
image: maksim64/hadoop-namenode:3.2.1
container_name: namenode
build: hadoop/namenode
restart: always
volumes:
- hadoop_namenode:/hadoop/dfs/name
ports:
- "9870:9870"
- "9010:9000"
hostname: namenode
env_file:
- ./hadoop/hadoop.env
environment:
- CLUSTER_NAME=test
- CORE_CONF_fs_defaultFS=hdfs://namenode:9000
datanode:
image: maskim64/hadoop-datanode:3.2.1
container_name: datanode
build: hadoop/datanode
restart: always
volumes:
- hadoop_datanode:/hadoop/dfs/data
ports:
- "9864:9864"
hostname: datanode
env_file:
- ./hadoop/hadoop.env
environment:
SERVICE_PRECONDITION: "namenode:9870"
CORE_CONF_fs_defaultFS: hdfs://namenode:9000
resourcemanager:
image: maskim64/hadoop-resourcemanager:3.2.1
container_name: resourcemanager
build: hadoop/resourcemanager
restart: always
ports:
- "8088:8088"
- "8032:8032"
hostname: resourcemanager
env_file:
- ./hadoop/hadoop.env
environment:
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
nodemanager1:
image: maksim64/hadoop-nodemanager:3.2.1
container_name: nodemanager
build: hadoop/nodemanager
restart: always
ports:
- "8042:8042"
hostname: nodemanager
env_file:
- ./hadoop/hadoop.env
environment:
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
historyserver:
image: maksim64/hadoop-historyserver:3.2.1
container_name: historyserver
build: hadoop/historyserver
restart: always
volumes:
- hadoop_historyserver:/hadoop/yarn/timeline
ports:
- "8188:8188"
hostname: historyserver
env_file:
- ./hadoop/hadoop.env
environment:
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
######## Spark configuration ########
spark-master:
image: maskim64/spark-master:3.1.1
container_name: spark-master
build: spark/master
depends_on:
- namenode
- datanode
ports:
- "8080:8080"
- "7077:7077"
environment:
- INIT_DAEMON_STEP=setup_spark
- CORE_CONF_fs_defaultFS=hdfs://namenode:9000
spark-worker-1:
image: maksim64/spark-worker:3.1.1
container_name: spark-worker-1
build: spark/worker
depends_on:
- spark-master
ports:
- "8081:8081"
environment:
- "SPARK_MASTER=spark://spark-master:7077"
- CORE_CONF_fs_defaultFS=hdfs://namenode:9000
######## Hive configuration ########
hive-server:
image: maksim64/hive:2.3.8
container_name: hive-server
build: hive
depends_on:
- namenode
- datanode
ports:
- "10000:10000"
- "10002:10002"
env_file:
- ./hive/hive.env
- ./hadoop/hadoop.env
environment:
SERVICE_PRECONDITION: "hive-metastore:9083"
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
hive-metastore:
image: maksim64/hive:2.3.8
container_name: hive-metastore
build: hive
command: /opt/hive/bin/hive --service metastore
ports:
- "9083:9083"
env_file:
- ./hive/hive.env
- ./hadoop/hadoop.env
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
hive-metastore-postgresql:
image: maksim64/hive-metastore-postgresql:3.1.0
build: hive-metastore-postgresql
container_name: hive-metastore-postgresql
######## Presto configuration ########
presto-coordinator:
image: shawnzhu/prestodb:0.181
container_name: presto-coordinator
ports:
- "8089:8089"
volumes:
hadoop_namenode:
hadoop_datanode:
hadoop_historyserver: