Skip to content

Commit

Permalink
branch-3.0: [opt](iceberg docker)Add health check for iceberg rest co…
Browse files Browse the repository at this point in the history
…ntainer #46767 (#47356)

Cherry-picked from #46767

Co-authored-by: Thearas <[email protected]>
  • Loading branch information
github-actions[bot] and Thearas authored Jan 27, 2025
1 parent 13db9d5 commit 8cc7d60
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,4 @@ echo "Script: create_view.hql executed in ${EXECUTION_TIME} seconds"
touch /mnt/SUCCESS

# Avoid container exit
while true; do
sleep 1
done
tail -f /dev/null
7 changes: 7 additions & 0 deletions docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@

export SPARK_MASTER_HOST=doris--spark-iceberg

# wait iceberg-rest start
while [[ ! $(curl -s --fail http://rest:8181/v1/config) ]]; do
sleep 1
done

set -ex

start-master.sh -p 7077
start-worker.sh spark://doris--spark-iceberg:7077
start-history-server.sh
Expand Down
38 changes: 26 additions & 12 deletions docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ services:
hostname: doris--spark-iceberg
build: spark/
depends_on:
- rest
- minio
- mc
rest:
condition: service_started
mc:
condition: service_completed_successfully
volumes:
- ./data/output/spark-warehouse:/home/iceberg/warehouse
- ./data/output/spark-notebooks:/home/iceberg/notebooks/notebooks
Expand All @@ -40,11 +41,11 @@ services:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
entrypoint: /bin/sh /mnt/scripts/entrypoint.sh
entrypoint: /bin/sh /mnt/scripts/entrypoint.sh
networks:
- doris--iceberg
healthcheck:
test: ["CMD", "ls", "/mnt/SUCCESS"]
test: ls /mnt/SUCCESS
interval: 5s
timeout: 120s
retries: 120
Expand All @@ -56,20 +57,28 @@ services:
POSTGRES_PASSWORD: 123456
POSTGRES_USER: root
POSTGRES_DB: iceberg
healthcheck:
test: [ "CMD-SHELL", "pg_isready -U root" ]
interval: 5s
timeout: 60s
retries: 120
volumes:
- ./data/input/pgdata:/var/lib/postgresql/data
networks:
- doris--iceberg

rest:
image: tabulario/iceberg-rest
image: tabulario/iceberg-rest:1.6.0
container_name: doris--iceberg-rest
ports:
- ${REST_CATALOG_PORT}:8181
volumes:
- ./data:/mnt/data
depends_on:
- postgres
postgres:
condition: service_healthy
minio:
condition: service_healthy
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
Expand All @@ -85,10 +94,15 @@ services:
entrypoint: /bin/bash /mnt/data/input/script/rest_init.sh

minio:
image: minio/minio
image: minio/minio:RELEASE.2025-01-20T14-49-07Z
container_name: doris--minio
ports:
- ${MINIO_API_PORT}:9000
healthcheck:
test: [ "CMD", "mc", "ready", "local" ]
interval: 10s
timeout: 60s
retries: 120
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
Expand All @@ -101,8 +115,9 @@ services:

mc:
depends_on:
- minio
image: minio/mc
minio:
condition: service_healthy
image: minio/mc:RELEASE.2025-01-17T23-25-50Z
container_name: doris--mc
environment:
- AWS_ACCESS_KEY_ID=admin
Expand All @@ -120,12 +135,11 @@ services:
/usr/bin/mc policy set public minio/warehouse;
echo 'copy data';
mc cp -r /mnt/data/input/minio/warehouse/* minio/warehouse/;
tail -f /dev/null
"

networks:
doris--iceberg:
ipam:
driver: default
config:
- subnet: 168.38.0.0/24
- subnet: 168.38.0.0/24
2 changes: 1 addition & 1 deletion docker/thirdparties/run-thirdparties-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -736,7 +736,7 @@ for compose in "${!pids[@]}"; do
if [ $status -ne 0 ] && [ $compose != "db2" ]; then
echo "docker $compose started failed with status $status"
echo "print start_${compose}.log"
cat start_${compose}.log
cat start_${compose}.log || true

echo ""
echo "print last 100 logs of the latest unhealthy container"
Expand Down

0 comments on commit 8cc7d60

Please sign in to comment.