feat(deps): add timeout setting for dependencies

Also modifies and replaces the SERVICE_DISABLE_HEALTHCHECK option
which can now be set by specifying HEALTH_TIMEOUT=0

CHANGELOG.md

@@ -22,6 +22,15 @@
 
 ---
 
+## unreleased
+
+* Feat: add `HEALTH_TIMEOUT` option to not hang forever if a dependency container
+  fails to start properly. Defaults to 60 (seconds). Also deprecates `SERVICE_DISABLE_HEALTCHECK`,
+  since that can now be done using `HEALTH_TIMEOUT=0`.
+  [#554](https://github.com/Kong/kong-pongo/pull/554).
+
+---
+
 ## 2.10.0 released 08-Feb-2024
 
 * Feat: add automatic reloads for interactive shells. This will watch plugin files as

README.md

@@ -84,7 +84,7 @@ Environment variables:
 
 Example usage:
   pongo run
-  KONG_VERSION=1.3.x pongo run -v -o gtest ./spec/02-access_spec.lua
+  KONG_VERSION=3.3.x pongo run -v -o gtest ./spec/02-access_spec.lua
   POSTGRES=10 KONG_IMAGE=kong-ee pongo run
   pongo down
 ```
@@ -356,9 +356,11 @@ a `.pongo/pongorc` file for a plugin that only needs Postgres and Redis:
 
 ### Disable Service Health Checks
 
-When unable to leverage container health checks, they can be disabled setting the environment variable `SERVICE_DISABLE_HEALTHCHECK=true`
-This will disable the service health checks for the Pongo services in the docker composer files
-for example
+When unable to leverage container health checks, they can be disabled setting the environment variable `HEALTH_TIMEOUT=0`.
+This will set the variable `SERVICE_DISABLE_HEALTHCHECK=true`, which can be used to disable the service health checks for
+the Pongo services in the docker composer files.
+
+For example:
 ```
     healthcheck:
       test:
@@ -370,7 +372,7 @@ for example
 ```
 To wait for the environment and run the tests one could run
 ```
-export SERVICE_DISABLE_HEALTHCHECK=true
+export HEALTH_TIMEOUT=0
 pongo up && sleep 10 && pongo run
 ```
 

assets/help/build.txt

@@ -35,6 +35,6 @@ latest Kong open source version.
 
 
 Example usage:
-  KONG_VERSION=1.3.x pongo build --force
+  KONG_VERSION=3.3.x pongo build --force
   KONG_VERSION=dev-ee pongo build
   KONG_IMAGE=custom-kong-ee pongo build

assets/help/pongo.txt

@@ -64,6 +64,6 @@ Environment variables:
 
 Example usage:
   pongo run
-  KONG_VERSION=1.3.x pongo run -v -o gtest ./spec/02-access_spec.lua
+  KONG_VERSION=3.3.x pongo run -v -o gtest ./spec/02-access_spec.lua
   POSTGRES_IMAGE=postgres:10 KONG_IMAGE=kong-ee pongo run
   pongo down

assets/help/run.txt

@@ -42,5 +42,5 @@ Environment variables:
 Example usage:
   pongo run
   KONG_VERSION=dev pongo run
-  KONG_VERSION=1.3.x pongo run -v -o TAP ./spec/02-access_spec.lua
+  KONG_VERSION=3.3.x pongo run -v -o TAP ./spec/02-access_spec.lua
   POSTGRES_IMAGE=postgres:10 KONG_IMAGE=custom-kong-ee pongo run

assets/help/up.txt

@@ -32,8 +32,8 @@ Environment variables:
   REDIS_IMAGE      the Redis dependency to use (default redis:6.2.6-alpine)
   SQUID_IMAGE      the Squid dependency to use (default sameersbn/squid:3.5.27-2)
   GRPCBIN_IMAGE    the Grpcbin dependency to use (default moul/grpcbin:latest)
-  SERVICE_DISABLE_HEALTHCHECK
-                   set to 'true' to disable dependency health checks globally
+  HEALTH_TIMEOUT   time in seconds to wait for dependencies to become healthy
+                   (default 60, set to 0 to disable health checks)
 Custom dependencies may have their own variables.
 
 

pongo.sh

@@ -129,6 +129,17 @@ function globals {
   DEVELOPMENT_CE_TAG="kong/kong:master-ubuntu"
 
 
+  # dependency health checks
+  if [[ -z $HEALTH_TIMEOUT ]]; then
+    export HEALTH_TIMEOUT=60
+  fi
+  if [[ $HEALTH_TIMEOUT -lt 0 ]]; then
+    export HEALTH_TIMEOUT=0
+  fi
+  if [[ $HEALTH_TIMEOUT -eq 0 ]]; then
+    export SERVICE_DISABLE_HEALTHCHECK=true
+  fi
+
   # Dependency image defaults
   if [[ -z $POSTGRES_IMAGE ]] && [[ -n $POSTGRES ]]; then
     # backward compat; POSTGRES replaced by POSTGRES_IMAGE
@@ -609,6 +620,10 @@ function compose {
 }
 
 
+# checks health status of a container. 2 args:
+# 1. container id (required)
+# 2. container name (optional, defaults to the id)
+# returns 0 (success) if healthy, 1 for all other states; starting, unhealthy, stopping, etc.
 function healthy {
   local iid=$1
   [[ -z $iid ]] && return 1
@@ -620,43 +635,55 @@ function healthy {
   fi
 
   if [[ "${SERVICE_DISABLE_HEALTHCHECK}" == "true" ]]; then
-    msg "Health checks disabled, won't wait for '$name' to be healthy"
     return 0
   fi
 
   local state
-  state=$(docker inspect "$iid")
+  state=$(docker inspect --format='{{.State.Health.Status}}' "$iid")
 
-  echo "$state" | grep \"Health\" &> /dev/null
-  if [[ ! $? -eq 0 ]]; then
-    # no healthcheck defined, assume healthy
-    msg "No health check available for '$name', assuming healthy"
+  if [ "$state" == "healthy" ]; then
     return 0
   fi
-
-  echo "$state" | grep \"healthy\" &> /dev/null
-  return $?
+  return 1
 }
 
 
+# takes a container name and returns its id
 function cid {
   compose ps -q "$1" 2> /dev/null
 }
 
 
+# Waits for a dependency to be healthy. 1 arg:
+# 1. dependency name
+# returns 0 (success) if healthy, throws an error if there was a timeout
 function wait_for_dependency {
   local iid
   local dep="$1"
 
+  if [[ "${SERVICE_DISABLE_HEALTHCHECK}" == "true" ]]; then
+    msg "Health checks disabled, won't wait for '$dep' to be healthy"
+    return 0
+  fi
+
   iid=$(cid "$dep")
 
-  if healthy "$iid" "$dep"; then return; fi
+  if healthy "$iid" "$dep"; then
+    return 0
+  fi
 
-  msg "Waiting for $dep"
+  msg "Waiting for '$dep' to become healthy"
 
-  while ! healthy "$iid" "$dep"; do
+  local timeout_count=$((HEALTH_TIMEOUT*2))
+  while [ $timeout_count -ge 0 ]; do
     sleep 0.5
+    if healthy "$iid" "$dep"; then
+      return 0
+    fi
+    timeout_count=$((timeout_count-1))
   done
+
+  err "Timeout waiting for '$dep' to become healthy"
 }