diff --git a/Dockerfile b/Dockerfile index af50778..1617bf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,13 @@ -# ----------------------------------------------------------------------------------------------- -# POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support -# using timescaledev/timescaledb-ha base image with: -# -# - OR specific ENV variables and a healthcheck added -# - PGDATA path set to match old Alpine image (for ease of DB migration) -# - POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration) -# - OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried -# out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists). -# This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes; -# migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value. -# NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed -# This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this -# themselves. -# -# -# -# -# timescale/timescaledb-ha image is ubuntu based and only currently supports amd64; they are -# working on ARM64 support in timescaledev/timescaledb-ha see: -# -# https://github.com/timescale/timescaledb-docker-ha/pull/355 -# -# See this issue for POSTGIS base image aarch64 support discussion: -# -# https://github.com/postgis/docker-postgis/issues/216 -# ------- ---------------------------------------------------------------------------------------- - -# TODO: Switch over to timescale/timescaledb-ha once arm64 supported -# We get POSTGIS and timescale+toolkit from this image + +ARG PG_MAJOR_PREVIOUS=14 +ARG PG_MAJOR=15 + FROM timescaledev/timescaledb-ha:pg15-multi as trimmed MAINTAINER support@openremote.io +ARG PG_MAJOR_PREVIOUS +ARG PG_MAJOR + USER root # Give postgres user the same UID and GID as the old alpine postgres image to simplify migration of existing DB @@ -54,12 +31,22 @@ RUN chmod +x /docker-entrypoint-initdb.d/* # Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint, # workdir and OR env defaults) +# Get multi all image +FROM timescaledev/timescaledb-ha:pg15-multi-all as trimmed-all + +ARG PG_MAJOR_PREVIOUS +ARG PG_MAJOR ## Create a smaller Docker image from the builder image FROM scratch COPY --from=trimmed / / -ARG PG_MAJOR=15 +ARG PG_MAJOR_PREVIOUS +ARG PG_MAJOR + +## Copy previous PG MAJOR version executable +COPY --from=trimmed-all /usr/lib/postgresql/${PG_MAJOR_PREVIOUS} /usr/lib/postgresql/${PG_MAJOR_PREVIOUS} +COPY --from=trimmed-all /usr/share/postgresql/${PG_MAJOR_PREVIOUS} /usr/share/postgresql/${PG_MAJOR_PREVIOUS} # Increment this to indicate that a re-index should be carried out on first startup with existing data; REINDEX can still be overidden # with OR_DISABLE_REINDEX=true @@ -86,9 +73,11 @@ ENV PGROOT=/var/lib/postgresql \ POSTGRES_DB=${POSTGRES_DB:-openremote} \ POSTGRES_USER=${POSTGRES_USER:-postgres} \ POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} \ + PG_MAJOR=$PG_MAJOR \ OR_REINDEX_COUNTER=${OR_REINDEX_COUNTER} \ OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false} \ - POSTGRES_MAX_CONNECTIONS=${POSTGRES_MAX_CONNECTIONS:-50} + POSTGRES_MAX_CONNECTIONS=${POSTGRES_MAX_CONNECTIONS:-50} \ + OR_DISABLE_AUTO_UPGRADE=${OR_DISABLE_AUTO_UPGRADE:-false} WORKDIR /var/lib/postgresql EXPOSE 5432 8008 8081 diff --git a/README.md b/README.md index 447e7e0..2e24570 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,28 @@ # Postgresql docker image [![build multirach postgresql Docker image and push to it dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml) -PostgreSQL docker image based on [Timescale's](https://www.timescale.com/) [timescaledev/timescaledb-ha](https://hub.docker.com/r/timescaledev/timescaledb-ha) docker image which contains post GIS and timescaleDB extensions. + POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support using `timescaledev/timescaledb-ha` base image with: + +- OR specific ENV variables and a healthcheck added +- PGDATA path set to match old Alpine image (for ease of DB migration) +- POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration) +- Auto upgrade of database with PG major version changes from previous PG major version; can be disabled using + OR_DISABLE_AUTO_UPGRADE=true. +- OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried + out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists). + This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes; + migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value. + NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed + This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this + themselves. + +`timescale/timescaledb-ha` image is ubuntu based and only currently supports amd64; they are working on ARM64 support in timescaledev/timescaledb-ha see: + +https://github.com/timescale/timescaledb-docker-ha/pull/355 + +See this issue for POSTGIS base image aarch64 support discussion: + +https://github.com/postgis/docker-postgis/issues/216 + +TODO: Switch over to timescale/timescaledb-ha once arm64 supported +We get POSTGIS and timescale+toolkit from this image diff --git a/or-entrypoint.sh b/or-entrypoint.sh index f481b5e..6d152af 100644 --- a/or-entrypoint.sh +++ b/or-entrypoint.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # THIS FILE IS FOR MIGRATION OF EXISTING DB TO TIMESCALEDB IMAGE AS TIMESCALE INIT SCRIPTS AREN'T RUN WHEN DB -# ALREADY EXISTS; IT ALSO DOES AN AUTOMATIC REINDEX OF THE DB WHEN OR_REINDEX_COUNTER changes TO SIMPLIFY MIGRATIONS +# ALREADY EXISTS; IT ALSO DOES AN AUTOMATIC REINDEX OF THE DB WHEN OR_REINDEX_COUNTER CHANGES TO SIMPLIFY MIGRATIONS +# IT ALSO AUTOMATICALLY HANDLES UPGRADING OF DATABASE AND DURING MAJOR VERSION CHANGES source /docker-entrypoint.sh docker_setup_env @@ -20,6 +21,168 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "timescaledb.telemetry_level=off" >> "$PGDATA/postgresql.conf" fi + ######################################################################################## + # Do upgrade checks - Adapted from https://github.com/pgautoupgrade/docker-pgautoupgrade + ######################################################################################## + + # Get the version of the PostgreSQL data files + DB_VERSION=$PG_MAJOR + if [ -s "${PGDATA}/PG_VERSION" ]; then + DB_VERSION=$(cat "${PGDATA}/PG_VERSION") + fi + + # Try and upgrade if needed + if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then + + echo "----------------------------------------------------------------------" + echo "Postgres major version is newer than the existing DB, performing auto upgrade..." + echo "----------------------------------------------------------------------" + + if [ ! -d "/usr/lib/postgresql/${DB_VERSION}" ]; then + echo "Postgres executable version '$DB_VERSION' is not included in this image so cannot auto upgrade" + exit 1 + fi + + # Check for presence of old/new directories, indicating a failed previous autoupgrade + echo "----------------------------------------------------------------------" + echo "Checking for left over artifacts from a failed previous autoupgrade..." + echo "----------------------------------------------------------------------" + OLD="${PGDATA}/old" + NEW="${PGDATA}/new" + if [ -d "${OLD}" ]; then + echo "*****************************************" + echo "Left over OLD directory found. Aborting." + echo "*****************************************" + exit 10 + fi + if [ -d "${NEW}" ]; then + echo "*****************************************" + echo "Left over NEW directory found. Aborting." + echo "*****************************************" + exit 11 + fi + echo "-------------------------------------------------------------------------------" + echo "No artifacts found from a failed previous autoupgrade. Continuing the process." + echo "-------------------------------------------------------------------------------" + + # Don't automatically abort on non-0 exit status, as that messes with these upcoming mv commands + set +e + + # Move the PostgreSQL data files into a subdirectory of the mount point + echo "---------------------------------------" + echo "Creating OLD temporary directory ${OLD}" + echo "---------------------------------------" + mkdir "${OLD}" + if [ ! -d "${OLD}" ]; then + echo "*********************************************************************" + echo "Creation of temporary directory '${OLD}' failed. Aborting completely" + echo "*********************************************************************" + exit 7 + fi + echo "--------------------------------------------" + echo "Creating OLD temporary directory is complete" + echo "--------------------------------------------" + + echo "-------------------------------------------------------" + echo "Moving existing data files into OLD temporary directory" + echo "-------------------------------------------------------" + mv -v "${PGDATA}"/* "${OLD}" + echo "-------------------------------------------------------------------" + echo "Moving existing data files into OLD temporary directory is complete" + echo "-------------------------------------------------------------------" + + echo "---------------------------------------" + echo "Creating NEW temporary directory ${NEW}" + echo "---------------------------------------" + mkdir "${NEW}" + if [ ! -d "${NEW}" ]; then + echo "********************************************************************" + echo "Creation of temporary directory '${NEW}' failed. Aborting completely" + echo "********************************************************************" + # With a failure at this point we should be able to move the old data back + # to its original location + mv -v "${OLD}"/* "${PGDATA}" + exit 8 + fi + echo "--------------------------------------------" + echo "Creating NEW temporary directory is complete" + echo "--------------------------------------------" + + echo "-----------------------------------------------------" + echo "Changing permissions of temporary directories to 0700" + echo "-----------------------------------------------------" + chmod 0700 "${OLD}" "${NEW}" + echo "---------------------------------------------------------" + echo "Changing permissions of temporary directories is complete" + echo "---------------------------------------------------------" + + # Return the error handling back to automatically aborting on non-0 exit status + set -e + + # Initialise the new PostgreSQL database directory + echo "--------------------------------------------------------------------------------------------------------------------" + echo "Initialising new database directory" + echo "--------------------------------------------------------------------------------------------------------------------" + initdb -D $PGDATA/new + echo "------------------------------------" + echo "New database directory initialisation complete" + echo "------------------------------------" + + # Change into the PostgreSQL database directory, to avoid a pg_upgrade error about write permissions + cd "${PGDATA}" + + # Run the pg_upgrade command itself + echo "---------------------------------------" + echo "Running pg_upgrade command, from $(pwd)" + echo "---------------------------------------" + pg_upgrade --link -b /usr/lib/postgresql/${DB_VERSION}/bin -B /usr/lib/postgresql/${PG_MAJOR}/bin -d $OLD -D $NEW + echo "--------------------------------------" + echo "Running pg_upgrade command is complete" + echo "--------------------------------------" + + # Move the new database files into place + echo "-----------------------------------------------------" + echo "Moving the upgraded database files to the active directory" + echo "-----------------------------------------------------" + mv -v "${NEW}"/* "${PGDATA}" + echo "-----------------------------------------" + echo "Moving the upgraded database files is complete" + echo "-----------------------------------------" + + # Re-use the pg_hba.conf and pg_ident.conf from the old data directory + echo "--------------------------------------------------------------" + echo "Copying the old pg_hba and pg_ident configuration files across" + echo "--------------------------------------------------------------" + cp -f "${OLD}/pg_hba.conf" "${OLD}/pg_ident.conf" "${PGDATA}" + echo "-------------------------------------------------------------------" + echo "Copying the old pg_hba and pg_ident configuration files is complete" + echo "-------------------------------------------------------------------" + + # Copy any reindex counter files + echo "--------------------------------------------------------------" + echo "Copying reindex files across" + echo "--------------------------------------------------------------" + cp -f ${OLD}/OR_REINDEX_* ${PGDATA} + echo "-------------------------------------------------------------------" + echo "Copying reindex files is complete" + echo "-------------------------------------------------------------------" + + # Remove the left over database files + echo "---------------------------------" + echo "Removing left over database files" + echo "---------------------------------" + set +e + rm -rf "${OLD}" "${NEW}" delete_old_cluster.sh + set -e + echo "---------------------------------------------" + echo "Removing left over database files is complete" + echo "---------------------------------------------" + + echo "**********************************************************" + echo "Automatic upgrade process finished with no errors reported" + echo "**********************************************************" + fi + # Do re-indexing check if [ "$OR_DISABLE_REINDEX" == 'true' ] || [ -z "$OR_REINDEX_COUNTER" ]; then echo "REINDEX check is disabled"