From bb1b4d38117a549b21d187b147e8f8b314207009 Mon Sep 17 00:00:00 2001
From: Aron Molnar <aron@syslifters.com>
Date: Fri, 30 Aug 2024 10:28:20 +0000
Subject: [PATCH] Fix update procedures and improve build caching

---
 .gitlab-ci.yml       | 28 ++++++++++++++++++----------
 CHANGELOG.md         |  7 ++++++-
 docs/docs/install.sh | 24 +++++++++++++-----------
 update.sh            |  6 +++---
 4 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6445e05ad..259c8ec0c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -180,19 +180,26 @@ build-release:
     - |
       if [[ $CI_COMMIT_TAG =~ '^prod-.*' ]]; then
         grep -qE "^## (v${VERSION_NUMBER})" CHANGELOG.md || exit 1
-        CACHE_RELEASE_TAG=test-$VERSION_NUMBER_LEADING_ZEROS
-        API_STATICS_IMAGE=$CI_REGISTRY_IMAGE/api-statics:$CACHE_RELEASE_TAG
+        TEST_CI_COMMIT_TAG=test-$VERSION_NUMBER_LEADING_ZEROS
       else
-        CACHE_RELEASE_TAG=$CI_COMMIT_SHORT_SHA
-        # Build and push api-statics
-        docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $CI_REGISTRY_IMAGE/pdfviewer-dev:$ISO_WEEK --cache-from $CI_REGISTRY_IMAGE/frontend-dev:$ISO_WEEK --target=api-statics --platform linux/amd64,linux/arm64 --push -t $API_STATICS_IMAGE .
+        TEST_CI_COMMIT_TAG=$CI_COMMIT_TAG
+        # Build/push depending stages
+        # rendering
+        docker buildx imagetools create $CI_REGISTRY_IMAGE/rendering:$ISO_WEEK --tag $CI_REGISTRY_IMAGE/rendering:$TEST_CI_COMMIT_TAG
+        # pdfviewer-dev
+        docker buildx imagetools create $CI_REGISTRY_IMAGE/pdfviewer-dev:$ISO_WEEK --tag $CI_REGISTRY_IMAGE/pdfviewer-dev:$TEST_CI_COMMIT_TAG
+        # frontend
+        docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $CI_REGISTRY_IMAGE/frontend-test:$CI_COMMIT_SHORT_SHA --cache-from $CI_REGISTRY_IMAGE/pdfviewer-dev:$TEST_CI_COMMIT_TAG --target=frontend --platform linux/amd64,linux/arm64 --push -t $CI_REGISTRY_IMAGE/frontend:$TEST_CI_COMMIT_TAG .
+        # api-statics
+        docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $CI_REGISTRY_IMAGE/frontend:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/pdfviewer-dev:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA --cache-from $CI_REGISTRY_IMAGE/rendering:$TEST_CI_COMMIT_TAG --target=api-statics --platform linux/amd64,linux/arm64 --push -t $CI_REGISTRY_IMAGE/api-statics:$TEST_CI_COMMIT_TAG .
       fi
     # Build containers
     - cd languagetool
-    - docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $CI_REGISTRY_IMAGE/languagetool:$CACHE_RELEASE_TAG --platform linux/amd64,linux/arm64 --push -t $CI_REGISTRY_IMAGE/languagetool:$CI_COMMIT_TAG .
+    - docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $CI_REGISTRY_IMAGE/languagetool:$TEST_CI_COMMIT_TAG --platform linux/amd64,linux/arm64 --push -t $CI_REGISTRY_IMAGE/languagetool:$CI_COMMIT_TAG .
     - cd ..
+    - TEST
     - export VERSION_NUMBER
-    - docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --build-arg VERSION="$VERSION_NUMBER" --cache-from $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA --cache-from $CI_REGISTRY_IMAGE:$CACHE_RELEASE_TAG --cache-from $API_STATICS_IMAGE --target=api --platform linux/amd64,linux/arm64 --push --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
+    - docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --build-arg VERSION="$VERSION_NUMBER" --cache-from $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA --cache-from $CI_REGISTRY_IMAGE/api-statics:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/frontend:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/pdfviewer-dev:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/rendering:$TEST_CI_COMMIT_TAG --target=api --platform linux/amd64,linux/arm64 --push --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
 
 release-gitlab-release:
   stage: release
@@ -222,7 +229,8 @@ release-prod:
     - VERSION_NUMBER_LEADING_ZEROS=$(echo "$CI_COMMIT_TAG" | sed -nr 's/^(prod|test|ltest)-([0-9]+\.[0-9]+([\.ab][0-9]+)?)$/\2/p')
     - VERSION_NUMBER=$(python3 -c "from packaging.version import Version;print(Version('${VERSION_NUMBER_LEADING_ZEROS}'))")
     - RELEASE_NOTES=$(awk "/## v${VERSION_NUMBER}.*/{include=1; next} /## v.*/{include=0} include && NF" CHANGELOG.md)
-    - sed -i '1s/^/SYSREPTOR_VERSION=${VERSION_NUMBER}\n/' deploy/.env
+    - TEST_CI_COMMIT_TAG=test-$VERSION_NUMBER_LEADING_ZEROS
+    - sed -i "1s/^/SYSREPTOR_VERSION=${VERSION_NUMBER}\n/" deploy/.env
     # Generate api notice file
     - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
     - CONTAINER_ID=$(docker create $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG)
@@ -231,7 +239,7 @@ release-prod:
     - rm -rf docs/docs/s docs/README.md docs/reporting_software.yml docs/wip docs/hooks.py dev .vscode api/.vscode
 
     # Build container with copyleft source code
-    - docker buildx build --build-arg BUILDKIT_INLINE_CACHE=1 --provenance false --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/api-statics:test-$VERSION_NUMBER_LEADING_ZEROS --target api-src --build-arg VERSION=$VERSION_NUMBER --target=api-src --platform linux/amd64,linux/arm64 --push --tag $DOCKER_HUB_IMAGE:$VERSION_NUMBER-src .
+    - docker buildx build --provenance false --build-arg BUILDKIT_INLINE_CACHE=1 --build-arg VERSION="$VERSION_NUMBER" --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/api-statics:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/frontend:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/pdfviewer-dev:$TEST_CI_COMMIT_TAG --cache-from $CI_REGISTRY_IMAGE/rendering:$TEST_CI_COMMIT_TAG --target=api-src --platform linux/amd64,linux/arm64 --push --tag $DOCKER_HUB_IMAGE:$VERSION_NUMBER-src .
     # Push containers to Docker Hub
     - docker buildx imagetools create $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --tag $DOCKER_HUB_IMAGE:$VERSION_NUMBER
     - docker buildx imagetools create $DOCKER_HUB_IMAGE:$VERSION_NUMBER --tag $DOCKER_HUB_IMAGE:latest
@@ -288,5 +296,5 @@ release-docs-on-prod-release:
   extends: .deploy_docs
   rules:
     - if: $CI_COMMIT_TAG =~ /^prod-.*/  # Run this job on prod deployments
-  needs: [release-prod]
+  needs: [release-prod, build-docs]
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f03526d4f..b3a2222e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Changelog
 
-## v2024.68 - 2024-08-29
+## v2024.69 - 2024-08-30
+* Fix install and update procedures
+* Fix error importing designs without ordering fields
+* Rename version history close button
+
+## v2024.68 - 2024-08-29 (pre-release)
 * Rework field definition data format. Warning: breaking API changes
 * Allow custom order of object field properties
 * Allow sorting combobox suggestions
diff --git a/docs/docs/install.sh b/docs/docs/install.sh
index ad001559f..a5d6f756a 100644
--- a/docs/docs/install.sh
+++ b/docs/docs/install.sh
@@ -57,8 +57,8 @@ then
     exit -4
 fi
 
-download_url=https://github.com/syslifters/sysreptor/releases/latest/download/source-prebuilt.tar.gz
-echo "Downloading SysReptor from $download_url ..."
+download_url=https://github.com/syslifters/sysreptor/releases/latest/download/setup.tar.gz
+echo "Downloading Docker Compose files from $download_url ..."
 curl -s -L --output sysreptor.tar.gz "$download_url"
 echo "Checking download..."
 if
@@ -100,10 +100,6 @@ else
         fi
     done
 
-    # Delete docker-compose.override.yml because that's needed for existing PRO installations only due to legacy reasons
-    # ...not for new installations
-    rm docker-compose.override.yml 2>/dev/null || true
-
     echo "Creating app.env..."
     cp app.env.example app.env
 
@@ -143,24 +139,30 @@ then
     source caddy/setup.sh || true  # do not exit on error
 fi
 
+# Delete docker-compose.override.yml because that's needed for existing PRO installations only due to legacy reasons
+# ...not for new installations
+rm docker-compose.override.yml 2>/dev/null || true
+
 echo "Creating docker volumes..."
 echo -n "Volume: "
 docker volume create sysreptor-db-data
 echo -n "Volume: "
 docker volume create sysreptor-app-data
 
-echo "Build and launch SysReptor via docker compose..."
-echo "We are downloading and installing all dependencies."
-echo "This may take a few minutes."
+echo "Launching SysReptor via docker compose..."
+echo "Downloading the Docker images may take a few minutes."
 
 if
+    source .env
+    export SYSREPTOR_VERSION
     ! docker compose up -d
 then
     echo "Ups. Something did not work while bringing up your containers."
     exit -2
 fi
 
-echo "Running migrations..."
+echo ""
+echo "Waiting for database setup..."
 while
     sleep 1
     ! echo '' | docker compose exec --no-TTY app python3 manage.py migrate --check 1>/dev/null 2>&1
@@ -206,7 +208,7 @@ do
     read -p "Copy your password now. Copied? [y/n]: " CONFIRM
     if [[ $CONFIRM == [nN] ]]
     then
-        echo "C'mon. Copy it. It's a good password."
+        echo "It's a good password. You will like it."
     fi
 done
 
diff --git a/update.sh b/update.sh
index 31dc1e7c9..f34872403 100755
--- a/update.sh
+++ b/update.sh
@@ -62,7 +62,7 @@ OLD_SYSREPTOR_VERSION=$SYSREPTOR_VERSION
 echo "Your current version is $OLD_SYSREPTOR_VERSION"
 
 echo "Checking if update is available..."
-version=`curl -s https://docs.sysreptor.org/latest.version`
+version=`curl -s https://docs.sysreptor.com/latest.version`
 if ! [[ $version =~ ^[0-9]{4}\.[0-9]+$ ]]
 then
     echo "Checking for new version failed."
@@ -100,8 +100,8 @@ echo "Copying your config files app.env and .env..."
 source "${sysreptor_directory}/deploy/.env"  # get new SYSREPTOR_VERSION
 NEW_SYSREPTOR_VERSION=$SYSREPTOR_VERSION
 cp "${backup_copy}/deploy/app.env" "${sysreptor_directory}/deploy/app.env"
-grep -v "^NEW_SYSREPTOR_VERSION=" "${backup_copy}/deploy/.env" > "${sysreptor_directory}/deploy/.env"
-sed -i '1s/^/NEW_SYSREPTOR_VERSION=${NEW_SYSREPTOR_VERSION}\n/' "${sysreptor_directory}/deploy/app.env"
+grep -v "^SYSREPTOR_VERSION=" "${backup_copy}/deploy/.env" > "${sysreptor_directory}/deploy/.env"
+sed -i "1s/^/SYSREPTOR_VERSION=${NEW_SYSREPTOR_VERSION}\n/" "${sysreptor_directory}/deploy/.env"
 
 if grep "sysreptor/docker-compose.yml" "${backup_copy}/deploy/docker-compose.yml" >/dev/null 2>&1
 then