Upgrade to version 1.74.0 (#46)

* mod.ts version update
* switching to the 1.74 from upstream
* Double timeout patching
* Removing old dependencies
* notes update and migration bump
* .gitignore update
* smarter Makefile

.gitignore

@@ -2,10 +2,7 @@
 synapse.s9pk
 image.tar
 base-image.tar
-homeserver.yaml
 .vscode/
-config.yaml
-homeserver1.yaml
 .DS_Store
 scripts/*.js
 docker-images

Dockerfile

@@ -1,153 +1,12 @@
-# Dockerfile to build the matrixdotorg/synapse docker images.
-#
-# Note that it uses features which are only available in BuildKit - see
-# https://docs.docker.com/go/buildkit/ for more information.
-#
-# To build the image, run `docker build` command from the root of the
-# synapse repository:
-#
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile .
-#
-# There is an optional PYTHON_VERSION build argument which sets the
-# version of python to build against: for example:
-#
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
-#
+FROM matrixdotorg/synapse:v1.74.0
 
-# Irritatingly, there is no blessed guide on how to distribute an application with its
-# poetry-managed environment in a docker image. We have opted for
-# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
-# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
-# In case we get bitten by those bugs in the future, the recommendations here might
-# be useful:
-#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
-#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
-
-
-
-ARG PYTHON_VERSION=3.10
-
-###
-### Stage 0: generate requirements.txt
-###
-FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
-
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt (and below for pip), to improve
-# rebuild speeds on slow connections.
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y git \
-    && rm -rf /var/lib/apt/lists/*
-
-# We install poetry in its own build stage to avoid its dependencies conflicting with
-# synapse's dependencies.
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --user "poetry==1.2.0"
-
-WORKDIR /synapse
-
-# Copy just what we need to run `poetry export`...
-COPY synapse/pyproject.toml synapse/poetry.lock /synapse/
-
-# If specified, we won't verify the hashes of dependencies.
-# This is only needed if the hashes of dependencies cannot be checked for some
-# reason, such as when a git repository is used directly as a dependency.
-ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION
-
-RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}
-
-###
-### Stage 1: builder
-###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
-
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
-# install the OS build deps
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y \
-    build-essential \
-    libffi-dev \
-    libjpeg-dev \
-    libpq-dev \
-    libssl-dev \
-    libwebp-dev \
-    libxml++2.6-dev \
-    libxslt1-dev \
-    openssl \
-    rustc \
-    zlib1g-dev \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-# To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project, so that this layer in the Docker cache can be
-# used while you develop on the source
-#
-# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
-COPY --from=requirements /synapse/requirements.txt /synapse/
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt
-
-# Copy over the rest of the synapse source code.
-COPY synapse/synapse /synapse/synapse/
-# ... and what we need to `pip install`.
-# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy
-# pyproject.toml here, ditching setup.py and MANIFEST.in.
-COPY synapse/pyproject.toml synapse/README.rst /synapse/
-
-# Install the synapse package itself.
-RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
-
-###
-### Stage 2: runtime
-###
-
-FROM docker.io/python:${PYTHON_VERSION}-slim as base-image
-
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && apt-get install -y \
-    curl \
-    gosu \
-    libjpeg62-turbo \
-    libpq5 \
-    libwebp6 \
-    xmlsec1 \
-    libjemalloc2 \
-    libssl-dev \
-    openssl \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --from=builder /install /usr/local
-COPY ./synapse/docker/start.py /start.py
-COPY ./synapse/docker/conf /conf
-
-EXPOSE 8008/tcp 8009/tcp 8448/tcp
-
-ENTRYPOINT ["/start.py"]
-
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-    CMD curl -fSs http://localhost:8008/health || exit 1
+ARG PLATFORM
+ENV YQ_VER v4.3.2
 
-FROM base-image
+RUN pip install --prefix="/install" --no-warn-script-location pyyaml
 
-RUN apt-get update \
-    && apt-get install -y \
+RUN apt-get update && \
+    apt-get install -y \
     tini \
     ca-certificates \
     nginx \
@@ -157,12 +16,16 @@ RUN apt-get update \
     privoxy \
     iproute2 \
     wget \
-    sqlite3
-
-ARG PLATFORM
-RUN wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.12.2/yq_linux_${PLATFORM} \
+    sqlite3; \
+    apt clean; \
+    rm -rf \
+    /config/* \
+    /tmp/* \
+    /var/lib/apt/lists/* \
+    /var/tmp/*
+
+RUN wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VER}/yq_linux_${PLATFORM} \
     && chmod a+x /usr/local/bin/yq
-RUN pip install --prefix="/install" --no-warn-script-location pyyaml
 
 ADD ./www /var/www
 ADD ./cert.conf /etc/ssl/cert.conf

Makefile

@@ -1,14 +1,17 @@
 PKG_ID := $(shell yq e ".id" manifest.yaml)
 PKG_VERSION := $(shell yq e ".version" manifest.yaml)
-SYNAPSE_SRC := $(shell find ./synapse)
 TS_FILES := $(shell find ./ -name \*.ts)
 
 .DELETE_ON_ERROR:
 
 all: verify
 
 install:
+ifeq (,$(wildcard ~/.embassy/config.yaml1))
+	@echo; echo "You must define \"host: http://embassy-server-name.local\" in ~/.embassy/config.yaml config file first"; echo
+else
 	embassy-cli package install $(PKG_ID).s9pk
+endif
 
 clean:
 	rm -f image.tar
@@ -22,23 +25,28 @@ verify: $(PKG_ID).s9pk
 	@echo "   Filesize: $(shell du -h $(PKG_ID).s9pk) is ready"
 
 $(PKG_ID).s9pk: manifest.yaml instructions.md icon.png LICENSE scripts/embassy.js docker-images/aarch64.tar docker-images/x86_64.tar
-	@if ! [ -z "$(ARCH)" ]; then cp docker-images/$(ARCH).tar image.tar; echo "* image.tar compiled for $(ARCH)"; fi
-	embassy-sdk pack
-
-docker-images/aarch64.tar: synapse/timeouts.patch Dockerfile docker_entrypoint.sh check-federation.sh priv-config-forward-all priv-config-forward-onion configurator.py $(shell find ./www)
+ifeq ($(ARCH),aarch64)
+	@echo "embassy-sdk: Preparing aarch64 package ..."
+else ifeq ($(ARCH),x86_64)
+	@echo "embassy-sdk: Preparing x86_64 package ..."
+else
+	@echo "embassy-sdk: Preparing Universal Package ..."
+endif
+	@embassy-sdk pack
+
+docker-images/aarch64.tar: Dockerfile docker_entrypoint.sh check-federation.sh priv-config-forward-all priv-config-forward-onion configurator.py $(shell find ./www)
+ifeq ($(ARCH),x86_64)
+else
 	mkdir -p docker-images
 	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --build-arg PLATFORM=arm64 --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --platform=linux/arm64 -o type=docker,dest=docker-images/aarch64.tar .
+endif
 
-docker-images/x86_64.tar: synapse/timeouts.patch Dockerfile docker_entrypoint.sh check-federation.sh priv-config-forward-all priv-config-forward-onion configurator.py $(shell find ./www)
+docker-images/x86_64.tar: Dockerfile docker_entrypoint.sh check-federation.sh priv-config-forward-all priv-config-forward-onion configurator.py $(shell find ./www)
+ifeq ($(ARCH),aarch64)
+else
 	mkdir -p docker-images
 	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --build-arg PLATFORM=amd64 --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --platform=linux/amd64 -o type=docker,dest=docker-images/x86_64.tar .
+endif
 
 scripts/embassy.js: $(TS_FILES)
 	deno bundle scripts/embassy.ts scripts/embassy.js
-
-# for running on a non-embassy amd64 linux server
-image-x86.tar: patch synapse/docker/Dockerfile $(SYNAPSE_SRC)
-	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build -f synapse/docker/Dockerfile --tag matrixdotorg/synapse:$(VERSION) --platform=linux/amd64 -o type=docker,dest=base-image.tar ./synapse
-
-synapse/timeouts.patch:
-	@cp timeouts.patch synapse/ && cd synapse && git apply -R --check < timeouts.patch 2>/dev/null; if [ "$?" = "0" ]; then git am < timeouts.patch; else echo "PATCH: Synapse appears to be patched ..."; fi && rm -f synapse/timeouts.patch

docker_entrypoint.sh

@@ -4,6 +4,7 @@ set -e
 
 export HOST_IP=$(ip -4 route list match 0/0 | awk '{print $3}')
 export TOR_ADDRESS=$(yq e '.tor-address' /data/start9/config.yaml)
+export TIMEOUT=20000
 echo "$HOST_IP   tor" >> /etc/hosts
 
 if ! [ -f /data/homeserver.yaml ]; then
@@ -83,6 +84,9 @@ python /configurator.py
 #Fixes and last minute config changes
 echo "enable_registration_without_verification: true" >> /data/homeserver.yaml
 echo "suppress_key_server_warning: true" >> /data/homeserver.yaml
+sed -i 's#timeout=10000#timeout='$TIMEOUT'#g' /usr/local/lib/python3*/site-packages/synapse/crypto/keyring.py
+sed -i 's#timeout=10000#timeout='$TIMEOUT'#g' /usr/local/lib/python3*/site-packages/synapse/federation/transport/client.py
+sed -i 's#timeout=10000#timeout='$TIMEOUT'#g' /usr/local/lib/python3*/site-packages/synapse/federation/federation_client.py
 nginx
 privoxy /etc/privoxy/config
 export https_proxy="127.0.0.1:8118"

manifest.yaml

@@ -1,10 +1,11 @@
 id: synapse
 title: Synapse
-version: 1.63.1.1
+version: 1.74.0
 release-notes: |
-  This one is Universal Package aka Fattie
-  * added support for x86_64 and aarch64 architecture
-  * fix for Reset First User action
+  * Update to latest upstream version
+  * Faster remote room joins [more info](https://github.com/matrix-org/synapse/pull/14473)
+  * Adjusted timeout
+  * Detailed changelog available [here](https://github.com/matrix-org/synapse/compare/v1.63.1...v1.74.0)
 license: apache
 wrapper-repo: https://github.com/Start9Labs/synapse-wrapper
 upstream-repo: https://github.com/Start9Labs/synapse
@@ -18,7 +19,6 @@ assets:
   license: LICENSE
   icon: icon.png
   instructions: instructions.md
-#  docker-images: image.tar
 main:
   type: docker
   image: main

scripts/procedures/migrations.ts

@@ -1,5 +1,4 @@
 import { compat, types as T } from "../deps.ts";
 
 export const migration: T.ExpectedExports.migration = compat.migrations
-    .fromMapping( {}, "1.63.1.1" );
-
+    .fromMapping( {}, "1.74.0" );