Merge branch 'prebuild-docker-images' into 'main'

Prebuild docker images

Closes #379 and #378

See merge request reportcreator/reportcreator!665

CHANGELOG.md

@@ -8,6 +8,7 @@
 * Improve error messages for decryption errors
 * Fix user.is_active checkbox not reactive in edit user page
 * Fix checkboxes not rendered as checked in PDF
+* Provide prebuilt Docker images
 * Fix chromium error while rendering PDFs
 
 

Dockerfile

@@ -1,39 +1,22 @@
-FROM node:20-alpine3.19 AS pdfviewer-dev
-
-# Add custom CA certificates
-ARG CA_CERTIFICATES=""
-RUN mkdir -p /usr/local/share/ca-certificates/ && \
-    echo "${CA_CERTIFICATES}" | tee -a /usr/local/share/ca-certificates/custom-user-cert.crt /etc/ssl/certs/ca-certificates.crt && \
-    apk add --no-cache ca-certificates && \
-    update-ca-certificates
+FROM --platform=$BUILDPLATFORM node:20-alpine3.19 AS pdfviewer-dev
 
 # Install dependencies
 WORKDIR /app/packages/pdfviewer/
 COPY packages/pdfviewer/package.json packages/pdfviewer/package-lock.json /app/packages/pdfviewer//
 RUN npm install
 
-FROM pdfviewer-dev AS pdfviewer
+FROM --platform=$BUILDPLATFORM pdfviewer-dev AS pdfviewer
 # Build JS bundle
 COPY packages/pdfviewer /app/packages/pdfviewer//
 RUN npm run build
 
 
 
 
-
-
-
-FROM node:20-alpine3.19 AS frontend-dev
+FROM --platform=$BUILDPLATFORM node:20-alpine3.19 AS frontend-dev
 
 ENV NODE_OPTIONS="--max-old-space-size=4096"
 
-# Add custom CA certificates
-ARG CA_CERTIFICATES=""
-RUN mkdir -p /usr/local/share/ca-certificates/ && \
-    echo "${CA_CERTIFICATES}" | tee -a /usr/local/share/ca-certificates/custom-user-cert.crt /etc/ssl/certs/ca-certificates.crt && \
-    apk add --no-cache ca-certificates && \
-    update-ca-certificates
-
 # Install dependencies
 WORKDIR /app/packages/markdown/
 COPY packages/markdown/package.json packages/markdown/package-lock.json /app/packages/markdown/
@@ -44,18 +27,18 @@ COPY frontend/package.json frontend/package-lock.json /app/frontend/
 RUN npm install
 
 
-FROM frontend-dev AS frontend-test
+FROM --platform=$BUILDPLATFORM frontend-dev AS frontend-test
 # Include source code
 COPY packages/markdown/ /app/packages/markdown/
 COPY frontend /app/frontend/
 COPY api/src/reportcreator_api/tasks/rendering/global_assets /app/frontend/src/assets/rendering/
 COPY --from=pdfviewer /app/packages/pdfviewer/dist/ /app/frontend/src/public/static/pdfviewer/
 
 # Test command
-CMD npm run test
+CMD ["npm", "run", "test"]
 
 
-FROM frontend-test AS frontend
+FROM --platform=$BUILDPLATFORM frontend-test AS frontend
 # Build JS bundle
 RUN npm run generate
 
@@ -65,14 +48,7 @@ RUN npm run generate
 
 
 
-FROM node:20-alpine3.19 AS rendering-dev
-
-# Add custom CA certificates
-ARG CA_CERTIFICATES=""
-RUN mkdir -p /usr/local/share/ca-certificates/ && \
-    echo "${CA_CERTIFICATES}" | tee -a /usr/local/share/ca-certificates/custom-user-cert.crt /etc/ssl/certs/ca-certificates.crt && \
-    apk add --no-cache ca-certificates && \
-    update-ca-certificates
+FROM --platform=$BUILDPLATFORM node:20-alpine3.19 AS rendering-dev
 
 # Install dependencies
 WORKDIR /app/packages/markdown/
@@ -84,7 +60,7 @@ COPY rendering/package.json rendering/package-lock.json /app/rendering/
 RUN npm install
 
 
-FROM rendering-dev AS rendering
+FROM --platform=$BUILDPLATFORM rendering-dev AS rendering
 # Include source code
 COPY rendering /app/rendering/
 COPY packages/markdown/ /app/packages/markdown/
@@ -96,14 +72,11 @@ RUN npm run build
 
 FROM python:3.12-slim-bookworm AS api-dev
 
-# Add custom CA certificates
-ARG CA_CERTIFICATES=""
-RUN echo "${CA_CERTIFICATES}" | tee -a /usr/local/share/ca-certificates/custom-user-cert.crt && \
-    update-ca-certificates && \
-    cat /etc/ssl/certs/* > /etc/ssl/certs/bundle.pem && \
-    pip config set global.cert /etc/ssl/certs/bundle.pem
-ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
-ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+# Get a list a preinstalled apt packages
+RUN mkdir /src && \
+    chown 1000:1000 /src && \
+    dpkg-query -W -f='${binary:Package}=${Version}\n' > /src/pre_installed.txt && \
+    echo "This image distributes binaries of copyleft licensed software. Please find the corresponding source code in our source-code distributing images (append `-src` to the image tags; e.g. syslifters/sysreptor:2024.58-src)." > /src/SOURCES.txt
 
 # Install system dependencies required by weasyprint and chromium
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -140,10 +113,15 @@ ENV PYTHONUNBUFFERED=on \
     PYTHONDONTWRITEBYTECODE=on \
     CHROMIUM_EXECUTABLE=/usr/lib/chromium/chromium \
     GHOSTSCRIPT_EXECUTABLE=/usr/bin/gs \
-    PATH=$PATH:/root/.local/bin
+    PATH=$PATH:/root/.local/bin \
+    REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+
+
 WORKDIR /app/api/
 COPY api/pyproject.toml api/poetry.lock /app/api/
-RUN pip install --no-cache poetry==1.8.3 && \
+RUN --mount=type=cache,target=/root/.cache/ \
+    pip install --no-cache poetry==1.8.3 && \
     poetry config virtualenvs.create false && \
     poetry install --no-cache --no-interaction --no-root
 
@@ -161,37 +139,29 @@ ENV VERSION=${VERSION} \
     SERVER_WORKERS=4 \
     PDF_RENDER_SCRIPT_PATH=/app/rendering/dist/bundle.js
 
-# Copy license and changelog
-COPY LICENSE CHANGELOG.md /app/
-COPY api/generate_notice.sh api/NOTICE /app/api/
-
 # Start server
 EXPOSE 8000
-CMD python3 manage.py migrate && \
-    gunicorn \
-        --bind=:8000 --worker-class=uvicorn.workers.UvicornWorker --workers=${SERVER_WORKERS} \
-        --max-requests=500 --max-requests-jitter=100 --graceful-timeout=300 \
-        reportcreator_api.conf.asgi:application
+CMD ["/bin/bash", "/app/api/start.sh"]
 
 
 
-FROM api-dev AS api-prebuilt
+FROM --platform=$BUILDPLATFORM api-dev AS api-prebuilt
 
 # Copy source code (including pre-build static files)
 COPY --chown=user:user api/src /app/api/
 COPY --chown=user:user rendering/dist /app/rendering/dist/
 
 
 
-FROM api-dev AS api-test
+FROM --platform=$BUILDPLATFORM api-dev AS api-test
 # Copy source code
 COPY --chown=user:user api/src /app/api/
 
 # Copy generated template rendering script
 COPY --from=rendering --chown=user:user /app/rendering/dist /app/rendering/dist/
 
 
-FROM api-test AS api
+FROM --platform=$BUILDPLATFORM api-test AS api-statics
 # Generate static frontend files
 # Post-process django files (for admin, API browser) and post-process them (e.g. add unique file hash)
 # Do not post-process nuxt files, because they already have hash names (and django failes to post-process them)
@@ -200,3 +170,24 @@ COPY --from=frontend /app/frontend/dist/index.html /app/frontend/dist/static/ /a
 RUN mv /app/api/frontend/static/index.html /app/api/frontend/index.html \
     && python3 manage.py collectstatic --no-input --no-post-process \
     && python3 -m whitenoise.compress /app/api/static/ map
+
+
+
+FROM api-test AS api
+COPY --from=api-statics /app/api/frontend/index.html /app/api/frontend/index.html
+COPY --from=api-statics /app/api/static/ /app/api/static/
+USER 0
+COPY --chown=1000:1000 api/generate_notice.sh api/download_sources.sh api/start.sh api/NOTICE /app/api/
+RUN /bin/bash /app/api/generate_notice.sh
+# Copy of changelog should be one of the last things to use cache for prod releases
+COPY LICENSE CHANGELOG.md /app/
+USER 1000
+
+
+
+
+FROM api AS api-src
+USER 0
+RUN dpkg-query -W -f='${binary:Package}=${Version}\n' > /src/post_installed.txt \
+    && bash /app/api/download_sources.sh
+USER 1000

INSTALL.md

@@ -0,0 +1 @@
+Find installation instructions at https://docs.sysreptor.com/setup/installation/

api/download_sources.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+set -e
+
+skip_license () {
+  copyright_file="$1"
+  if test -f "$copyright_file"; then
+    license=`cat "$copyright_file" | grep -m 1 "^License:" | cut -d" " -f 2`
+    if test -n "$license" && echo "$skip_licenses" | grep -wq "$license"; then
+      return 0
+    fi
+  fi
+  return 1
+}
+
+fetch_source() {
+  package="$1"
+  package_name=`echo $package | cut -d= -f1 | cut -d: -f1`
+  if echo "$skip_packages" | grep -wq "$package_name"; then
+    echo "Skipping $package"
+    return
+  fi
+  if skip_license "/usr/share/doc/$package_name/copyright"; then
+    echo "$package_name (installed) $license license doesn't require source code distribution"
+    return
+  fi
+  echo "Fetching $package"
+  mkdir "$package"
+  cd "$package"
+  apt-get source "$package" --download-only > /dev/null 2>&1
+  cd ..
+  echo "Done $package"
+}
+
+skip_packages="fonts-dejavu-core
+libgl1
+libglvnd0
+libglx0
+libtiff6
+fontconfig
+fontconfig-config
+fonts-noto-color-emoji
+libavahi-client3
+libavahi-common-data
+libavahi-common3
+libcairo-gobject2
+libcairo2
+libdrm-amdgpu1
+libdrm-common
+libdrm-intel1
+libdrm-nouveau2
+libdrm-radeon1
+libdrm2
+libfontconfig1
+libfontenc1
+libice6
+libpciaccess0
+libpixman-1-0
+libsensors-config
+libsensors5
+libsm6
+libwebp7
+libx11-6
+libx11-data
+libx11-xcb1
+libxau6
+libxaw7
+libxcb-dri2-0
+libxcb-dri3-0
+libxcb-glx0
+libxcb-present0
+libxcb-randr0
+libxcb-render0
+libxcb-shape0
+libxcb-shm0
+libxcb-sync1
+libxcb-xfixes0
+libxcb1
+libxcomposite1
+libxcursor1
+libxdamage1
+libxdmcp6
+libxext6
+libxfixes3
+libxft2
+libxi6
+libxinerama1
+libxkbcommon0
+libxkbfile1
+libxmu6
+libxmuu1
+libxpm4
+libxrandr2
+libxrender1
+libxshmfence1
+libxslt1.1
+libxt6
+libxtst6
+libxv1
+libxxf86dga1
+libxxf86vm1
+unzip
+x11-common
+x11-utils
+xfonts-encodings
+xfonts-utils
+xkb-data"
+
+skip_licenses="MIT
+curl
+CC-BY-SA-3.0
+LGPL-2+
+BSD-3-clause
+OFL-1.1
+SIL-1.1
+CC0
+LPGL-2.1+
+APACHE-2-LLVM-EXCEPTIONS
+Apache-2.0-with-GPL2-LGPL2-Exception
+BSD-2-clause
+Expat
+FTL
+BSD-BY-LC-NE
+Apache-2.0
+zlib
+BSD-2
+libpng
+PostgreSQL
+X11
+MIT-1"
+
+cd /src
+sed -i "s/^Types: deb$/Types: deb deb-src/" /etc/apt/sources.list.d/debian.sources
+apt-get update > /dev/null 2>&1
+
+# Diff with previously installed packages
+for package in `grep -Fxv -f pre_installed.txt post_installed.txt`; do
+  fetch_source "$package"
+done
+
+apt-get clean > /dev/null 2>&1
+rm -rf /var/lib/apt/lists/*

api/generate_notice.sh

@@ -25,11 +25,11 @@ ignore="$ignore freetype2"
 ignore="$ignore cwe"
 ignore="$ignore randomcolor"
 
+export PATH="/home/user/.local/bin:$PATH"
 pip3 install pip-licenses
 pip-licenses --allow-only "$allow_only" --ignore-packages $ignore >/dev/null
 pip-licenses -l --no-license-path -f plain-vertical --no-version --ignore-packages $ignore > NOTICE
 
-
 # Those packages do not include valid license files
 webencodings=$(cat << EOF
 BSD License

api/start.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+if [[ -n "$CA_CERTIFICATES" ]]; then
+    echo "${CA_CERTIFICATES}" >> /usr/local/share/ca-certificates/custom-user-cert.crt
+    update-ca-certificates
+fi
+
+python3 manage.py migrate
+gunicorn --bind=:8000 \
+         --worker-class=uvicorn.workers.UvicornWorker \
+         --workers=${SERVER_WORKERS} \
+         --max-requests=500 \
+         --max-requests-jitter=100 \
+         --graceful-timeout=300 \
+         reportcreator_api.conf.asgi:application

deploy/.env

@@ -1,3 +1 @@
-# Do not modify. This file is automatically generated.
-# Make changes to host.env instead.
-SYSREPTOR_VERSION=dev
+# BIND_PORT="127.0.0.1:8000:8000"

deploy/languagetool/docker-compose.yml

@@ -2,10 +2,7 @@
 # Changes will be overwritten.
 services:
   languagetool:
-    build: 
-      context: ../../languagetool
-      args:
-        CA_CERTIFICATES: ${SYSREPTOR_CA_CERTIFICATES-}
+    image: syslifters/sysreptor-languagetool:${SYSREPTOR_VERSION:-latest}
     container_name: 'sysreptor-languagetool'
     init: true
     environment: