Introduce rolling docker config / secret update concept 🎉 🚀 (#952)

* fixes

* update comment

scripts/common.Makefile

@@ -207,9 +207,39 @@ clean-default: .check_clean ## Cleans all outputs
 	export DEPLOYMENT_API_DOMAIN_TESTING_CAPTURE_TRAEFIK_RULE='${DEPLOYMENT_API_DOMAIN_TESTING_CAPTURE_TRAEFIK_RULE}'; \
 	export DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS='${DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS}'; \
 	export DOLLAR='$$'; \
+	$(if $(STACK_NAME),export STACK_NAME='$(STACK_NAME)';) \
 	set +o allexport; \
 	envsubst < $< > .env
 
+ifdef STACK_NAME
+
+.PHONY: prune-docker-stack-configs-default
+prune-docker-stack-configs-default: ## Clean all unused stack configs
+	@# Since the introduction of rolling docker config updates old
+	@# [docker config] versions are kept. This target removes them
+	@# https://github.com/docker/cli/issues/203
+	@#
+	@# This should be run before stack update in order to
+	@# keep previous config version for potential rollback
+	@#
+	@# This will not clean "external" configs. To achieve this extend
+	@# this target in related Makefiles.
+	@#
+	@# Long live Kubernetes ConfigMaps!
+
+	@for id in $$(docker config ls --filter "label=com.docker.stack.namespace=${STACK_NAME}" --format '{{.ID}}'); do \
+	    docker config rm "$$id" >/dev/null 2>&1 || true; \
+	done
+
+.PHONY: prune-docker-stack-secrets-default
+prune-docker-stack-secrets-default: ## Clean all unused stack secrets
+	@# Same as for configs
+
+	@for id in $$(docker secret ls --filter "label=com.docker.stack.namespace=${STACK_NAME}" --format '{{.ID}}'); do \
+	    docker secret rm "$$id" >/dev/null 2>&1 || true; \
+	done
+
+endif
 
 # Helpers -------------------------------------------------
 .PHONY: .init
@@ -253,13 +283,16 @@ venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environmen
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \
+	--customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
 endif

scripts/j2cli_global_filters.py

@@ -0,0 +1,19 @@
+def sha256file(filepath):
+    import hashlib
+
+    _hash = hashlib.sha256()
+
+    # https://stackoverflow.com/a/22058673/12124525
+    with open(filepath, "rb") as f:
+        while True:
+            data = f.read(65536)
+            if not data:
+                break
+
+            _hash.update(data)
+
+    return _hash.hexdigest()
+
+
+def substring(value, start, end):
+    return value[start:end]

services/appmotion_gateway/.gitignore

@@ -1,2 +1,2 @@
-!docker-compose.yml
+docker-compose.yml
 *.secret

services/appmotion_gateway/Makefile

@@ -1,30 +1,37 @@
 .DEFAULT_GOAL := help
 
-
-
 # Internal VARIABLES ------------------------------------------------
 # STACK_NAME defaults to name of the current directory. Should not to be changed if you follow GitOps operating procedures.
 STACK_NAME = $(notdir $(shell pwd))
 TEMP_COMPOSE=.stack.${STACK_NAME}.yaml
 REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 
+DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES = .api_env.secret \
+										  prune-docker-stack-configs  \
+										  prune-docker-stack-secrets
+
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 .PHONY: up-aws ## Deploys stack on aws
-up-aws: .init .env ${TEMP_COMPOSE}-aws .api_env.secret
+up-aws: .init .env ${TEMP_COMPOSE}-aws ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES}
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME}
 
 .PHONY: up-master ## Deploys stack on master
-up-master: .init .env ${TEMP_COMPOSE}-master .api_env.secret
+up-master: .init .env ${TEMP_COMPOSE}-master ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES}
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
 
 .PHONY: up-local ## Deploys stack on local
-up-local: .init .env ${TEMP_COMPOSE}-local .api_env.secret
+up-local: .init .env ${TEMP_COMPOSE}-local ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES}
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
 
 # Helpers -------------------------------------------------
 
+docker-compose.yml: .env .api_env.secret
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
+	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
+	rm docker-compose.yml.unlinted >/dev/null 2>&1;
+
 .PHONY: ${TEMP_COMPOSE}-aws
 ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
@@ -37,6 +44,5 @@ ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml
 ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.local.yml > $@
 
-
 .api_env.secret: .env template.api_env ## resolves '.api_env.secret' using '.env'
 	@set -o allexport; source $<; set +o allexport; envsubst < $(word 2,$^) > $@

services/appmotion_gateway/docker-compose.yml → services/appmotion_gateway/docker-compose.yml.j2

@@ -103,6 +103,7 @@ configs:
     # SEE https://docs.docker.com/compose/compose-file/05-services/#configs
     api_env_config:
         file: ./.api_env.secret
+        name: ${STACK_NAME}_api_env_config_{{ "./.api_env.secret" | sha256file | substring(0,10) }} # rolling update on content change
 
 volumes:
     # SEE https://docs.docker.com/compose/compose-file/07-volumes/

services/appmotion_gateway/template.env

@@ -20,5 +20,5 @@ DEPLOYMENT_FQDNS_APPMOTION_CAPTURE_TRAEFIK_RULE='${DEPLOYMENT_FQDNS_APPMOTION_CA
 APPMOTION_NETWORK=${APPMOTION_NETWORK}
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
-SWARM_STACK_NAME=${SWARM_STACK_NAME}
+STACK_NAME=${STACK_NAME}
 MACHINE_FQDN=${MACHINE_FQDN}