From 4c6f45c454ab6f75513167555cbc915e6b5fb818 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 23 Jun 2021 07:34:47 +0200 Subject: [PATCH] Rework packaging. Build .deb packages for amd64, arm64v8 and arm32v7 Package building now takes place within Docker containers. For all named architectures/platforms, distribution packages are built for Debian stretch and buster as well as Ubuntu bionic and focal. --- .dockerignore | 1 - .gitignore | 1 + Makefile | 222 +++----------- README.rst | 3 +- .../development/releasing/packaging.rst | 62 +++- fabfile.py | 57 ---- packaging/builder/fpm-package | 26 +- packaging/dockerfiles/Dockerfile.all.kotori | 70 ----- .../dockerfiles/Dockerfile.debian.baseline | 31 -- .../dockerfiles/debian-baseline.dockerfile | 67 ++++ .../dockerfiles/debian-package.dockerfile | 70 +++++ ...ile.hub.kotori => docker-image.dockerfile} | 34 ++- .../docker-image.dockerfile.dockerignore | 6 + packaging/etc/logrotate.conf | 10 + packaging/tasks.mk | 102 +++++++ packaging/wheels/build.sh | 96 ++++++ packaging/wheels/upload.sh | 10 + requirements-release.txt | 10 +- setup.py | 2 + tasks/README.rst | 28 ++ tasks/__init__.py | 5 + tasks/packaging/__init__.py | 5 + tasks/packaging/docker.py | 288 ++++++++++++++++++ tasks/packaging/environment.py | 85 ++++++ tasks/packaging/model.py | 134 ++++++++ tasks/packaging/ospackage.py | 153 ++++++++++ tasks/util.py | 31 ++ test/util.py | 4 +- 28 files changed, 1232 insertions(+), 381 deletions(-) delete mode 100644 fabfile.py delete mode 100644 packaging/dockerfiles/Dockerfile.all.kotori delete mode 100644 packaging/dockerfiles/Dockerfile.debian.baseline create mode 100644 packaging/dockerfiles/debian-baseline.dockerfile create mode 100644 packaging/dockerfiles/debian-package.dockerfile rename packaging/dockerfiles/{Dockerfile.hub.kotori => docker-image.dockerfile} (67%) create mode 100644 packaging/dockerfiles/docker-image.dockerfile.dockerignore create mode 100644 packaging/etc/logrotate.conf create mode 100644 packaging/tasks.mk create mode 100755 packaging/wheels/build.sh create mode 100755 packaging/wheels/upload.sh create mode 100644 tasks/README.rst create mode 100644 tasks/__init__.py create mode 100644 tasks/packaging/__init__.py create mode 100644 tasks/packaging/docker.py create mode 100644 tasks/packaging/environment.py create mode 100644 tasks/packaging/model.py create mode 100644 tasks/packaging/ospackage.py create mode 100644 tasks/util.py diff --git a/.dockerignore b/.dockerignore index 9e85d1a3..45e95fa5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,3 @@ !packaging !README.rst !CHANGES.rst -!dist diff --git a/.gitignore b/.gitignore index 30d5164e..8aa518c8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.deb /doc/build /dist +/pkgs /tmp /var .coverage diff --git a/Makefile b/Makefile index 6cd1c60b..bff8d21b 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,10 @@ # -*- coding: utf-8 -*- # (c) 2014-2021 Andreas Motl -# ============ -# Main targets -# ============ - -# ------------- +# ============= # Configuration -# ------------- +# ============= $(eval venv := .venv) $(eval pip := $(venv)/bin/pip) @@ -18,26 +14,38 @@ $(eval nosetests := $(venv)/bin/nosetests) $(eval bumpversion := $(venv)/bin/bumpversion) $(eval twine := $(venv)/bin/twine) $(eval sphinx := $(venv)/bin/sphinx-build) -$(eval fab := $(venv)/bin/fab) +$(eval invoke := $(venv)/bin/invoke) + + +# ===== +# Setup +# ===== -# Setup Python virtualenv +# Setup Python virtualenv. setup-virtualenv: @test -e $(python) || python3 -m venv $(venv) +# Install requirements for building the documentation. virtualenv-docs: setup-virtualenv @$(pip) --quiet install --requirement=requirements-docs.txt +# Install requirements for development. virtualenv-dev: setup-virtualenv @$(pip) install --upgrade --requirement=requirements-test.txt @$(pip) install --upgrade --editable=.[daq,daq_geospatial,export,scientific,firmware] +# Install requirements for releasing. +install-releasetools: setup-virtualenv + @$(pip) install --quiet --requirement=requirements-release.txt --upgrade + + # ======= # Release # ======= -# -# Release this piece of software + +# Release this piece of software. # Uses the fine ``bumpversion`` utility. # # Synopsis:: @@ -47,156 +55,6 @@ virtualenv-dev: setup-virtualenv release: bumpversion push sdist pypi-upload - -publish-sdist: sdist - # publish Python Eggs to eggserver - # TODO: use localshop or one of its sisters - rsync -auv --progress ./dist/kotori-*.tar.gz workbench@packages.elmyra.de:/srv/packages/organizations/elmyra/foss/htdocs/python/kotori/ - - - - -# ========================================== -# packaging -# ========================================== - - -# Build baseline images and packages. - -package-baseline-images: - $(fab) build-docker-debian-baseline-images - $(fab) build-docker-ubuntu-baseline-images - -package-all: check-version - - # amd64 - # TODO: Also build "standard" flavors for amd64 to reduce footprint of Docker images. - $(MAKE) package-debian flavor=full dist=stretch arch=amd64 version=$(version) - $(MAKE) package-debian flavor=full dist=buster arch=amd64 version=$(version) - $(MAKE) package-debian flavor=full dist=bionic arch=amd64 version=$(version) - $(MAKE) package-debian flavor=full dist=focal arch=amd64 version=$(version) - - # armv7hf - $(MAKE) package-debian flavor=standard dist=stretch arch=armv7hf version=$(version) - $(MAKE) package-debian flavor=standard dist=buster arch=armv7hf version=$(version) - - # aarch64 - $(MAKE) package-debian flavor=standard dist=stretch arch=aarch64 version=$(version) - $(MAKE) package-debian flavor=standard dist=buster arch=aarch64 version=$(version) - - -# Build and publish debian package with flavor. -# Hint: Should be run on an appropriate build slave matching the deployment platform. - -# Synopsis:: -# -# # amd64 -# make package-debian flavor=full dist=buster arch=amd64 version=0.22.0 -# -# # armhf -# make package-debian flavor=standard dist=buster arch=armhf version=0.22.0 -# - -package-debian: check-flavor-options deb-build-$(flavor) publish-debian - - -deb-build-minimal: - $(MAKE) deb-build name=kotori-minimal features=daq - -deb-build-standard: - $(MAKE) deb-build name=kotori-standard features=daq,export - -deb-build-standard-binary: - $(MAKE) deb-build name=kotori-standard-binary features=daq,export,daq_binary - -deb-build-full: - $(MAKE) deb-build name=kotori features=daq,daq_geospatial,export,plotting,firmware,scientific - - -deb-build: check-build-options - - @# https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux - @# https://en.wikipedia.org/wiki/ANSI_escape_code - $(eval RED := "\033[0;31m") - $(eval YELLOW := \033[0;33m\033[1m) - $(eval NC := \033[0m) - - @echo "Building package $(YELLOW)$(name)$(NC) version $(YELLOW)$(version)$(NC) with features $(YELLOW)$(features)$(NC)" - - # Build Python virtualenv and Linux distribution package - @#docker build --tag daq-tools/kotori-build-arm32v7:$(version) --build-arg VERSION=$(version) --build-arg NAME=$(name) --build-arg FEATURES=$(features) --file packaging/dockerfiles/Dockerfile.kotori.arm32v7 . - @#docker build --tag daq-tools/kotori-build-arm32v7:$(version) --build-arg BASE_IMAGE=hiveeyes/arm32v7-baseline --build-arg VERSION=$(version) --build-arg NAME=$(name) --build-arg FEATURES=$(features) --file packaging/dockerfiles/Dockerfile.kotori.arm32v7 . - - docker build --tag daq-tools/kotori-build-$(arch):$(version) --build-arg BASE_IMAGE=daq-tools/$(dist)-$(arch)-baseline:latest --build-arg DISTRIBUTION=$(dist) --build-arg VERSION=$(version) --build-arg NAME=$(name) --build-arg FEATURES=$(features) --file packaging/dockerfiles/Dockerfile.all.kotori . - - # Extract Debian package - docker container rm -f finalize; true - docker container create --name finalize daq-tools/kotori-build-$(arch):$(version) - docker container cp finalize:/dist/$(name)_$(version)-1~$(dist)_$(arch).deb ./dist/ - - docker container rm -f finalize - - -publish-debian: - # Publish all Debian packages - rsync -auv --progress ./dist/kotori*$(version)*.deb workbench@packages.elmyra.de:/srv/packages/organizations/elmyra/foss/aptly/public/incoming/ - - - -# ================= -# Docker Hub images -# ================= - -package-dockerhub-image: check-version - docker build --tag daqzilla/kotori:$(version) --build-arg version=$(version) --file packaging/dockerfiles/Dockerfile.hub.kotori . - docker tag daqzilla/kotori:$(version) daqzilla/kotori:nightly - @#docker tag daqzilla/kotori:$(version) daqzilla/kotori:latest - - -# ================= -# Packaging helpers -# ================= - -check-version: - @if test "$(version)" = ""; then \ - echo "ERROR: 'version' not set"; \ - exit 1; \ - fi - -check-flavor-options: - @if test "$(flavor)" = ""; then \ - echo "ERROR: 'flavor' not set, try 'make package-debian flavor={minimal,standard,full}'"; \ - exit 1; \ - fi - - -check-build-options: - @if test "$(dist)" = ""; then \ - echo "ERROR: 'dist' not set"; \ - exit 1; \ - fi - @if test "$(arch)" = ""; then \ - echo "ERROR: 'arch' not set"; \ - exit 1; \ - fi - @if test "$(name)" = ""; then \ - echo "ERROR: 'name' not set"; \ - exit 1; \ - fi - @if test "$(features)" = ""; then \ - echo "ERROR: 'features' not set"; \ - exit 1; \ - fi - @if test "$(version)" = ""; then \ - echo "ERROR: 'version' not set"; \ - exit 1; \ - fi - - - -# =============== -# Utility targets -# =============== bumpversion: install-releasetools check-bump-options $(bumpversion) $(bump) @@ -209,10 +67,6 @@ sdist: pypi-upload: install-releasetools twine upload --skip-existing --verbose dist/*.tar.gz -install-releasetools: setup-virtualenv - @$(pip) install --quiet --requirement=requirements-release.txt --upgrade - - check-bump-options: @if test "$(bump)" = ""; then \ echo "ERROR: 'bump' not set, try 'make release bump={patch,minor,major}'"; \ @@ -221,13 +75,17 @@ check-bump-options: -# ========================================== -# environment -# ========================================== -# -# Miscellaneous tools: -# Software tests, Documentation builder, Virtual environment builder -# +# ========= +# Packaging +# ========= + +include packaging/tasks.mk + + + +# ============== +# Software tests +# ============== .PHONY: test pytest: virtualenv-dev @@ -252,15 +110,18 @@ test-coverage: virtualenv-dev --with-coverage --cover-package=kotori --cover-tests \ --cover-html --cover-html-dir=coverage/html --cover-xml --cover-xml-file=coverage/coverage.xml + + +# ============= +# Documentation +# ============= + +# Build Sphinx documentation. docs-html: virtualenv-docs touch doc/source/index.rst SPHINXBUILD="`pwd`/$(sphinx)" SPHINXOPTS="-j auto" make --directory=doc html - -# ========================================== -# ptrace.getkotori.org -# ========================================== - +# Upload media assets. Images, videos, etc. # Don't commit media assets (screenshots, etc.) to the repository. # Instead, upload them to https://ptrace.getkotori.org/ ptrace_target := root@ptrace.getkotori.org:/srv/www/organizations/daq-tools/ptrace.getkotori.org/htdocs/ @@ -286,9 +147,12 @@ check-ptrace-options: fi +# ============== +# Infrastructure +# ============== + +start-foundation-services: + docker-compose up -# ========================================== -# infrastructure -# ========================================== mongodb-start: mongod --dbpath=./var/lib/mongodb/ --smallfiles diff --git a/README.rst b/README.rst index a7394c61..175c2bab 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Kotori :target: https://github.com/daq-tools/kotori/actions?workflow=Tests .. image:: https://img.shields.io/pypi/pyversions/kotori.svg - :target: https://python.org + :target: https://pypi.org/project/kotori/ .. image:: https://img.shields.io/pypi/v/kotori.svg :target: https://pypi.org/project/kotori/ @@ -47,6 +47,7 @@ Kotori :target: https://pypi.org/project/kotori/ .. image:: https://img.shields.io/pypi/l/kotori.svg + :alt: License :target: https://pypi.org/project/kotori/ - **Infrastructure**: diff --git a/doc/source/development/releasing/packaging.rst b/doc/source/development/releasing/packaging.rst index 6c7ca6f1..3fbea648 100644 --- a/doc/source/development/releasing/packaging.rst +++ b/doc/source/development/releasing/packaging.rst @@ -1,9 +1,9 @@ .. _kotori-package: .. _kotori-build: -############# -Run packaging -############# +######### +Packaging +######### .. highlight:: bash @@ -12,10 +12,15 @@ Run packaging Prerequisites ************* -Prepare baseline images:: +Prepare baseline Docker images:: make package-baseline-images +Prepare dependency wheel packages:: + + ./packaging/wheels/build.sh + ./packaging/wheels/upload.sh + *************** Debian packages @@ -23,25 +28,54 @@ Debian packages Build packages for all targets:: - make package-all + make package-all version=0.26.6 Build individual packages for Debian and Ubuntu:: # amd64 - make package-debian flavor=full dist=stretch arch=amd64 version=0.24.5 - make package-debian flavor=full dist=buster arch=amd64 version=0.24.5 - make package-debian flavor=full dist=bionic arch=amd64 version=0.24.5 + make package-debian flavor=full dist=stretch arch=amd64 version=0.26.6 + make package-debian flavor=full dist=buster arch=amd64 version=0.26.6 + make package-debian flavor=full dist=bionic arch=amd64 version=0.26.6 + + # arm64v8 + make package-debian flavor=standard dist=stretch arch=arm64v8 version=0.26.6 + make package-debian flavor=standard dist=buster arch=arm64v8 version=0.26.6 + + # arm32v7 + make package-debian flavor=standard dist=stretch arch=arm32v7 version=0.26.6 + make package-debian flavor=standard dist=buster arch=arm32v7 version=0.26.6 - # armv7hf - make package-debian flavor=standard dist=stretch arch=armv7hf version=0.24.5 - make package-debian flavor=standard dist=buster arch=armv7hf version=0.24.5 ************* Docker images ************* -:: - make package-dockerhub-image version=0.24.5 + +Authenticate with Docker Hub +============================ + +We need to do both:: + + # Run ``docker login`` to be able to regularly push images. docker login - docker push daqzilla/kotori + + # Set environment variables because the ``manifest-tool`` requires that. + export DOCKER_USERNAME=johndoe + export DOCKER_PASSWORD=supersecret + + +Build and publish Docker images +=============================== + +Invoke:: + + make package-docker-images version=0.26.6 + +Run basic QA checks:: + + make package-docker-qa tag=0.26.6 + +Designate specific version as ``latest``:: + + make package-docker-link version=0.26.6 tag=latest diff --git a/fabfile.py b/fabfile.py deleted file mode 100644 index 85531847..00000000 --- a/fabfile.py +++ /dev/null @@ -1,57 +0,0 @@ -import os - -from fabric import task - -docker_image_version = "0.8.0" -python_version = "3.8" -debian_distributions = ["buster", "stretch"] # "bullseye" -debian_architectures = ["amd64", "aarch64", "armv7hf"] -ubuntu_builds = [ - {"distribution": "bionic", "image": "ubuntu:bionic-20210118"}, - {"distribution": "focal", "image": "ubuntu:focal-20210119"}, -] - - -@task -def build_docker_debian_baseline_images(context): - for distribution in debian_distributions: - for architecture in debian_architectures: - commands = [ - f""" - docker build \ - --tag daq-tools/{distribution}-{architecture}-baseline:{docker_image_version} \ - --build-arg BASE_IMAGE=balenalib/{architecture}-debian-python:{python_version}-{distribution}-build - \ - < packaging/dockerfiles/Dockerfile.debian.baseline - """, - f""" - docker tag daq-tools/{distribution}-{architecture}-baseline:{docker_image_version} daq-tools/{distribution}-{architecture}-baseline:latest - """ - ] - run_commands(commands) - - -@task -def build_docker_ubuntu_baseline_images(context): - architecture = "amd64" - for build in ubuntu_builds: - distribution = build["distribution"] - image = build["image"] - commands = [ - f""" - docker build \ - --tag daq-tools/{distribution}-{architecture}-baseline:{docker_image_version} \ - --build-arg BASE_IMAGE={image} - \ - < packaging/dockerfiles/Dockerfile.debian.baseline - """, - f""" - docker tag daq-tools/{distribution}-{architecture}-baseline:{docker_image_version} daq-tools/{distribution}-{architecture}-baseline:latest - """ - ] - run_commands(commands) - - -def run_commands(commands): - for command in commands: - command = command.strip() - print(command) - os.system(command) diff --git a/packaging/builder/fpm-package b/packaging/builder/fpm-package index 87cd6f26..414f44b4 100755 --- a/packaging/builder/fpm-package +++ b/packaging/builder/fpm-package @@ -9,14 +9,19 @@ # # Synopsis:: # -# fpm-package kotori stretch 0.21.1 +# fpm-package kotori 0.21.1 {stretch,buster} {amd64,aarch64,armv7hf} +# +# Example:: +# +# fpm-package kotori 0.21.1 stretch amd64 # NAME=$1 -DISTRIBUTION=$2 -VERSION=$3 +VERSION=$2 +DISTRIBUTION=$3 +ARCHITECTURE=$4 -echo "Building package $NAME-$VERSION for ${DISTRIBUTION}" +echo "Building package $NAME-$VERSION for ${DISTRIBUTION} on ${ARCHITECTURE}" # Build Debian package mkdir -p ./dist @@ -32,12 +37,14 @@ fpm \ --no-deb-use-file-permissions \ --no-python-obey-requirements-txt \ --no-python-dependencies \ - --deb-recommends "libatlas3-base, libopenblas-base, liblapack3, libhdf5-100, libnetcdf-c++4, libnetcdf11" \ - --deb-recommends "liblzo2-2, libbz2-1.0, fonts-humor-sans" \ - --deb-suggests "influxdb, mosquitto, mosquitto-clients, grafana, mongodb" \ + --depends "python3" \ + --deb-recommends "liblzo2, libbz2, libblosc1, libhdf5-100, libhdf5-103, libnetcdf-c++4" \ + --deb-recommends "libatlas3-base, libopenblas-base, libblis2, libmkl-rt, liblapack3" \ + --deb-recommends "fonts-humor-sans" \ + --deb-suggests "mosquitto, mosquitto-clients, grafana, influxdb, mongodb, logrotate" \ --provides "kotori" \ --provides "${NAME}" \ - --maintainer "andreas.motl@elmyra.de" \ + --maintainer "Andreas Motl " \ --license "AGPL 3, EUPL 1.2" \ --deb-changelog CHANGES.rst \ --deb-meta-file README.rst \ @@ -55,14 +62,13 @@ fpm \ /opt/kotori \ ./etc/production.ini=/etc/kotori/kotori.ini \ ./etc/examples/=/etc/kotori/examples \ + ./packaging/etc/logrotate.conf=/etc/logrotate.d/kotori \ ./packaging/systemd/kotori.service=/usr/lib/systemd/system/kotori.service # Optionally # --debug \ -#--depends "python3" \ - # Might use again when building from feature branches or other references #--deb-field 'Branch: $(branch) Commit: $(commit)' \ #version := $(shell python setup.py --version) diff --git a/packaging/dockerfiles/Dockerfile.all.kotori b/packaging/dockerfiles/Dockerfile.all.kotori deleted file mode 100644 index 77915d46..00000000 --- a/packaging/dockerfiles/Dockerfile.all.kotori +++ /dev/null @@ -1,70 +0,0 @@ -# -# Build Kotori package using "fpm". -# -# Synopsis: -# -# make package-debian flavor=full dist=buster arch=amd64 version=0.24.3 -# - -ARG BASE_IMAGE - - - -# ================= -# Create virtualenv -# ================= -FROM ${BASE_IMAGE} AS python-environment - -ARG PREFIX=/opt/kotori - -RUN python -m venv --system-site-packages --copies ${PREFIX} - - - -# ====================== -# Install Python package -# ====================== -FROM python-environment AS install-kotori - -ARG VERSION -ARG FEATURES - -ARG PREFIX=/opt/kotori -ARG pip=${PREFIX}/bin/pip - -RUN echo hello - -# Install Kotori from PyPI. -ENV TMPDIR=/var/tmp -RUN $pip install kotori[${FEATURES}]==${VERSION} --upgrade - - - -# =========================== -# Create distribution package -# =========================== -FROM install-kotori AS package-kotori - -ARG DISTRIBUTION -ARG VERSION -ARG NAME -ARG FEATURES - -ARG PREFIX=/opt/kotori - - -# Counter "ValueError: bad marshal data (unknown type code)" -# coming from manipulation through "virtualenv-tools"". -RUN find ${PREFIX} -name '*.pyc' -delete -RUN find ${PREFIX} -name '__pycache__' -delete - -# Copy over specific resources required for package building. -WORKDIR / -COPY README.rst README.rst -COPY CHANGES.rst CHANGES.rst -COPY etc etc -COPY packaging packaging - -# Build package. -ENV TMPDIR=/var/tmp -RUN ./packaging/builder/fpm-package "${NAME}" "${DISTRIBUTION}" "${VERSION}" diff --git a/packaging/dockerfiles/Dockerfile.debian.baseline b/packaging/dockerfiles/Dockerfile.debian.baseline deleted file mode 100644 index 8bc7d41a..00000000 --- a/packaging/dockerfiles/Dockerfile.debian.baseline +++ /dev/null @@ -1,31 +0,0 @@ -ARG BASE_IMAGE - -FROM ${BASE_IMAGE} AS debian-build - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get upgrade - -# Build foundation and header files -RUN apt-get install --yes --no-install-recommends apt-utils -RUN apt-get install --yes --no-install-recommends \ - git nano \ - build-essential pkg-config libssl-dev libffi-dev libyaml-dev libpng-dev libfreetype6-dev #\ - #python3 python3-dev python3-setuptools python3-virtualenv virtualenv - -# Scipy, Numpy, Matplotlib and PyTables -#RUN apt-get install --yes --install-recommends \ -# python3-requests python3-openssl python3-cryptography python3-certifi \ -# python3-pandas python3-numpy python3-scipy python3-matplotlib python3-tables python3-netcdf4 - -#RUN apt-get install --yes --no-install-recommends \ -# gfortran libatlas-dev libopenblas-dev liblapack-dev libcoarrays-dev \ -# libhdf5-dev libnetcdf-dev liblzo2-dev libbz2-dev \ - - -FROM debian-build - -# FPM -RUN apt-get install --yes --no-install-recommends \ - ruby ruby-dev && \ - gem install fpm --version 1.12.0 diff --git a/packaging/dockerfiles/debian-baseline.dockerfile b/packaging/dockerfiles/debian-baseline.dockerfile new file mode 100644 index 00000000..d3a78132 --- /dev/null +++ b/packaging/dockerfiles/debian-baseline.dockerfile @@ -0,0 +1,67 @@ +# +# Build baseline Docker images for building distribution packages. +# +# https://getkotori.org/docs/development/releasing/packaging.html +# +# Synopsis: +# +# make package-baseline-images +# + +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} AS debian-fpm + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get upgrade --yes + +# On `arm32v7/debian:buster-slim`, Ruby croaks with certificate errors. +# https://guides.rubygems.org/ssl-certificate-update/ +# https://stackoverflow.com/questions/40549268/certificate-verify-failed-while-using-http-rubygems-org-instead-of-https +RUN test -e /etc/debian_version && test $(uname --machine) = "armv7l" && \ + apt-get install --yes wget && \ + wget --output-document=/usr/lib/ssl/cert.pem \ + https://raw.githubusercontent.com/rubygems/rubygems/master/lib/rubygems/ssl_certs/rubygems.org/GlobalSignRootCA_R3.pem \ + ; true + +# Install essential dependencies. +RUN apt-get install --yes --no-install-recommends \ + inetutils-ping nano git \ + build-essential pkg-config libffi-dev \ + ruby ruby-dev + +# Install fpm (Effing package management). +RUN gem install fpm --version=1.12.0 + + +FROM debian-fpm + +# Build foundation and header files +#RUN apt-get install --yes --no-install-recommends apt-utils +RUN apt-get install --yes --no-install-recommends \ + python3 python3-dev python3-venv \ + libssl-dev libyaml-dev libpng-dev libfreetype6-dev + + #python3-setuptools python3-virtualenv virtualenv + +# NumPy, pandas, Matplotlib, PyTables, PyNetCDF and more +RUN # apt-get install --yes --install-recommends \ + \ + # baseline \ + python3-requests python3-openssl python3-cryptography python3-certifi \ + \ + # extra: export \ + python3-pandas \ + \ + # extra: plotting \ + python3-matplotlib \ + \ + # extra: scientific \ + python3-tables python3-netcdf4 libatlas3-base + + # python3-scipy + +#RUN apt-get install --yes --no-install-recommends \ +# gfortran libatlas-base-dev libopenblas-dev liblapack-dev libcoarrays-dev \ +# libhdf5-dev libnetcdf-dev liblzo2-dev libbz2-dev libblosc-dev diff --git a/packaging/dockerfiles/debian-package.dockerfile b/packaging/dockerfiles/debian-package.dockerfile new file mode 100644 index 00000000..6f840aa4 --- /dev/null +++ b/packaging/dockerfiles/debian-package.dockerfile @@ -0,0 +1,70 @@ +# +# Build Debian package using "fpm". +# +# https://getkotori.org/docs/development/releasing/packaging.html +# https://getkotori.org/docs/setup/linux-debian.html +# +# Synopsis: +# +# make package-debian flavor=full dist=buster arch=amd64 version=0.26.6 +# + +ARG BASE_IMAGE + + +# ======================================= +# Install package into Python environment +# ======================================= +FROM ${BASE_IMAGE} AS python-environment + +ARG PYTHON_PACKAGE=kotori +ARG PREFIX=/opt/kotori +ARG VERSION +ARG FEATURES + +# Create Python virtualenv +RUN python3 -m venv ${PREFIX} + +ARG pip=${PREFIX}/bin/pip + +# Fix `pip` re. `PIP_EXTRA_INDEX_URL`. +# https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1822842 +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=837764 +RUN ${pip} install --upgrade --force-reinstall "pip<21" wheel + +# Announce extra PyPI repository containing pre-built packages for `aarch64` (arm64v8) and `armv7l` (arm32v7). +ENV PIP_EXTRA_INDEX_URL=https://packages.elmyra.de/elmyra/foss/python/ + +# Install package from PyPI. +ENV TMPDIR=/var/tmp +RUN ${pip} install ${PYTHON_PACKAGE}[${FEATURES}]==${VERSION} --prefer-binary --upgrade + + + +# =========================== +# Create distribution package +# =========================== +FROM python-environment AS package + +ARG NAME +ARG VERSION +ARG DISTRIBUTION +ARG ARCHITECTURE + +ARG PREFIX=/opt/kotori + + +# Counter "ValueError: bad marshal data (unknown type code)". +RUN find ${PREFIX} -name '*.pyc' -delete +RUN find ${PREFIX} -name '__pycache__' -delete + +# Copy over specific resources required for package building. +WORKDIR / +COPY README.rst README.rst +COPY CHANGES.rst CHANGES.rst +COPY etc etc +COPY packaging packaging + +# Build package. +ENV TMPDIR=/var/tmp +RUN ./packaging/builder/fpm-package "${NAME}" "${VERSION}" "${DISTRIBUTION}" "${ARCHITECTURE}" diff --git a/packaging/dockerfiles/Dockerfile.hub.kotori b/packaging/dockerfiles/docker-image.dockerfile similarity index 67% rename from packaging/dockerfiles/Dockerfile.hub.kotori rename to packaging/dockerfiles/docker-image.dockerfile index a62370da..cdfb97e8 100644 --- a/packaging/dockerfiles/Dockerfile.hub.kotori +++ b/packaging/dockerfiles/docker-image.dockerfile @@ -1,24 +1,34 @@ # # Build Docker image for publishing on Docker Hub. -# https://getkotori.org/docs/setup/debian-quickstart.html +# +# https://getkotori.org/docs/development/releasing/packaging.html +# https://getkotori.org/docs/setup/docker.html # # Synopsis: # -# make build-dockerhub-image version=0.24.5 +# make package-dockerhub-image version=0.24.5 # -#FROM debian:buster-slim -#FROM debian:bullseye-slim -FROM balenalib/amd64-debian-python:3.8-buster-run +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} + +ARG VERSION +ARG RELEASE_DATE -LABEL maintainer="Andreas Motl " +ARG PACKAGE_FILE -ARG version +LABEL version="${VERSION}" +LABEL release-date="${RELEASE_DATE}" +LABEL description="Kotori is a data acquisition, processing and graphing toolkit for humans" +LABEL maintainer="Andreas Motl " + +#ARG package_name=kotori_${VERSION}-1~buster_amd64.deb #RUN timedatectl set-ntp true # Tweak: Use `kotori*.deb` from local filesystem. -COPY ./dist/*.deb /tmp +COPY ./${PACKAGE_FILE} /tmp # Ramping up. RUN \ @@ -38,10 +48,10 @@ RUN \ # apt-get update && apt-get install --yes --install-recommends systemd- influxdb- grafana- mongodb- mosquitto- mosquitto-clients- && \ \ # Install Kotori. -#apt-get update && apt-get install --yes --install-recommends kotori=${version}* && \ - +#apt-get update && apt-get install --yes --install-recommends kotori=${VERSION}* && \ +\ # Tweak: Use `kotori*.deb` from local filesystem. -apt-get install --yes --install-recommends /tmp/kotori*.deb && \ +apt-get install --yes --install-recommends /tmp/$(basename ${PACKAGE_FILE}) && \ ln -s /opt/kotori/bin/kotori /usr/local/sbin/ && \ \ # Tearing down. @@ -53,4 +63,4 @@ ln -s /opt/kotori/bin/kotori /usr/local/sbin/ && \ EXPOSE 24642 -CMD [ "/opt/kotori/bin/kotori" ] +CMD [ "kotori" ] diff --git a/packaging/dockerfiles/docker-image.dockerfile.dockerignore b/packaging/dockerfiles/docker-image.dockerfile.dockerignore new file mode 100644 index 00000000..9e85d1a3 --- /dev/null +++ b/packaging/dockerfiles/docker-image.dockerfile.dockerignore @@ -0,0 +1,6 @@ +* +!etc +!packaging +!README.rst +!CHANGES.rst +!dist diff --git a/packaging/etc/logrotate.conf b/packaging/etc/logrotate.conf new file mode 100644 index 00000000..8a429743 --- /dev/null +++ b/packaging/etc/logrotate.conf @@ -0,0 +1,10 @@ +# Rotate log files weekly and keep them for one year. +/var/log/kotori/kotori.log { + su kotori kotori + weekly + rotate 52 + missingok + dateext + copytruncate + compress +} diff --git a/packaging/tasks.mk b/packaging/tasks.mk new file mode 100644 index 00000000..3ea5f279 --- /dev/null +++ b/packaging/tasks.mk @@ -0,0 +1,102 @@ +# *************** +# Packaging tasks +# *************** + + +# ===================== +# Distribution packages +# ===================== + +# Build Docker baseline images for packaging. + +package-baseline-images: + $(invoke) packaging.environment.baseline-images + + +# Build all operating system distribution packages. +# +# Synopsis:: +# +# # amd64 +# make package-all version=0.26.6 + +package-all: check-version + $(invoke) packaging.ospackage.run --version=$(version) + + +# Build Debian package. +# +# Synopsis:: +# +# # amd64 +# make package-debian flavor=full dist=buster arch=amd64 version=0.26.6 +# +# # arm64v8 +# make package-debian flavor=standard dist=buster arch=arm64v8 version=0.26.6 +# +# # arm32v7 +# make package-debian flavor=standard dist=buster arch=arm32v7 version=0.26.6 + +package-debian: + $(invoke) packaging.ospackage.deb --version=$(version) --flavor=$(flavor) --distribution=$(dist) --architecture=$(arch) + + +publish-debian: + # Publish all Debian packages + rsync -auv --progress ./dist/kotori*$(version)*.deb workbench@packages.elmyra.de:/srv/packages/organizations/elmyra/foss/aptly/public/incoming/ + + + +# ================= +# Docker Hub images +# ================= + +package-docker-images: + $(invoke) packaging.docker.images --version=$(version) + +package-docker-link: + $(invoke) packaging.docker.link --version=$(version) --tag=$(tag) + +package-docker-qa: + $(invoke) packaging.docker.qa --tag=$(tag) + + + +# ================= +# Packaging helpers +# ================= + +check-version: + @if test "$(version)" = ""; then \ + echo "ERROR: 'version' not set"; \ + exit 1; \ + fi + +check-flavor-options: + @if test "$(flavor)" = ""; then \ + echo "ERROR: 'flavor' not set, try 'make package-debian flavor={minimal,standard,full}'"; \ + exit 1; \ + fi + + +check-build-options: + @if test "$(dist)" = ""; then \ + echo "ERROR: 'dist' not set"; \ + exit 1; \ + fi + @if test "$(arch)" = ""; then \ + echo "ERROR: 'arch' not set"; \ + exit 1; \ + fi + @if test "$(name)" = ""; then \ + echo "ERROR: 'name' not set"; \ + exit 1; \ + fi + @if test "$(features)" = ""; then \ + echo "ERROR: 'features' not set"; \ + exit 1; \ + fi + @if test "$(version)" = ""; then \ + echo "ERROR: 'version' not set"; \ + exit 1; \ + fi diff --git a/packaging/wheels/build.sh b/packaging/wheels/build.sh new file mode 100755 index 00000000..9e7c1fec --- /dev/null +++ b/packaging/wheels/build.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Program to build wheel packages for uploading them to a package repository. + +# When seeing troubles with arm32v7, maybe populate `/lib/binfmt.d`. +# https://github.com/docker/for-linux/issues/56#issuecomment-502263368 + + +function invoke_docker() { + image=$1 + flavor=$2 + docker run -it --rm --volume=$(pwd)/packaging/wheels:/tools --volume=$(pwd)/pkgs:/pkgs --volume=$(pwd):/src ${image} /tools/build.sh ${flavor} +} + +function invoke_build() { + + flavor=$1 + + if [ $flavor = "full" ]; then + extras="daq,daq_geospatial,export,plotting,firmware,scientific" + elif [ $flavor = "standard" ]; then + extras="daq,daq_geospatial,export" + else + echo "ERROR: Package flavor '${flavor}' unknown or not implemented" + exit 1 + fi + + set -o xtrace + + # Install required Debian packages. + export DEBIAN_FRONTEND=noninteractive + apt-get update && apt-get upgrade --yes + apt-get install --yes \ + python3 python3-dev python3-pip libssl-dev libffi-dev \ + liblzo2-dev libbz2-dev libblosc-dev libhdf5-dev libnetcdf-dev \ + gfortran libatlas-base-dev libopenblas-dev liblapack-dev libcoarrays-dev + apt-get install --yes libblis-dev libmkl-dev || true + + # Use `wheel` package still compatible with Python 3.5. + pip3 install --upgrade --force-reinstall "pip<21" wheel + + # Don't use Rust for building `cryptography`. + export CRYPTOGRAPHY_DONT_BUILD_RUST=1 + + # If wheels are already in repository, they don't have to be built again. + export PIP_EXTRA_INDEX_URL=https://packages.elmyra.de/elmyra/foss/python/ + + # Install Kotori, thus building all wheel packages. + pip3 install /src[${extras}] --prefer-binary --upgrade --verbose + + # Copy compiled wheel packages to /pkgs path. + find /root/.cache/pip/wheels -iname '*cp*.whl' -exec cp {} /pkgs/ \; +} + +# Define Docker image names. +full_images=( + daq-tools/stretch-amd64-baseline + daq-tools/buster-amd64-baseline + daq-tools/bionic-amd64-baseline + daq-tools/focal-amd64-baseline +) +standard_images=( + daq-tools/stretch-arm64v8-baseline + daq-tools/buster-arm64v8-baseline + daq-tools/stretch-arm32v7-baseline + daq-tools/buster-arm32v7-baseline +) + +function main() { + + # How to determine if a process runs inside Docker? + # https://stackoverflow.com/a/25518345 + if [ -f /.dockerenv ]; then + echo "I'm inside matrix ;(" + invoke_build $1 + + else + echo "I'm living in real world!" + + # Build wheels for `full` packages. + for imagename in ${full_images[@]}; do + echo "Building wheels on ${imagename}" + invoke_docker ${imagename} full; + done + + # Build wheels for `standard` packages. + for imagename in ${standard_images[@]}; do + echo "Building wheels on ${imagename}" + invoke_docker ${imagename} standard; + done + + fi + +} + +main $@ diff --git a/packaging/wheels/upload.sh b/packaging/wheels/upload.sh new file mode 100755 index 00000000..8ec43ff6 --- /dev/null +++ b/packaging/wheels/upload.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +pkgnames=$(ls pkgs/*.whl | perl -pe 's!.+/(.+?)-.*$!$1!' | uniq) + +for pkgname in ${pkgnames} +do + cmd="rsync -auv ./pkgs/${pkgname}* root@pulp.cicer.de:/srv/packages/organizations/elmyra/foss/htdocs/python/${pkgname}/" + echo $cmd + $cmd +done diff --git a/requirements-release.txt b/requirements-release.txt index f7a0cef4..fd71e46d 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1,4 +1,6 @@ -bump2version==1.0.1 -twine==3.3.0 -keyring==22.0.1 -Fabric==2.6.0 +bump2version>=1,<2 +twine>=3,<4 +keyring>=20,<23 +invoke>=1,<2 +requests>=2,<3 +sh>=1,<2 diff --git a/setup.py b/setup.py index 391c1278..3e24340c 100644 --- a/setup.py +++ b/setup.py @@ -148,6 +148,8 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", "License :: OSI Approved :: GNU Affero General Public License v3", "License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)", "Development Status :: 5 - Production/Stable", diff --git a/tasks/README.rst b/tasks/README.rst new file mode 100644 index 00000000..706b0245 --- /dev/null +++ b/tasks/README.rst @@ -0,0 +1,28 @@ +############# +Project tasks +############# + + +************ +Introduction +************ + +This directory contains common tasks used for project maintenance. Mostly, it +contains tasks related to release package and image building. + +To get an overview about the implemented tasks, invoke:: + + invoke --list + + +********* +Packaging +********* + +Tasks for building distribution packages and images using Docker. Building +packages and images is a three-stage process: + +1. Build Docker baseline images. +2. Build distribution packages using those baseline images. +3. Build Docker images using those packages. +4. Run basic tests on Docker images. diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 00000000..6a048a1d --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,5 @@ +from invoke import Collection + +import tasks.packaging + +namespace = Collection(tasks.packaging) diff --git a/tasks/packaging/__init__.py b/tasks/packaging/__init__.py new file mode 100644 index 00000000..eed0649e --- /dev/null +++ b/tasks/packaging/__init__.py @@ -0,0 +1,5 @@ +from invoke import Collection + +from . import docker, environment, ospackage + +namespace = Collection(environment, ospackage, docker) diff --git a/tasks/packaging/docker.py b/tasks/packaging/docker.py new file mode 100644 index 00000000..05fa709e --- /dev/null +++ b/tasks/packaging/docker.py @@ -0,0 +1,288 @@ +import dataclasses +import os +from datetime import datetime +from pathlib import Path +from typing import List + +import requests +import sh +from invoke import Context + +from tasks.packaging.model import DockerImageRecipe, PackageSpecification +from tasks.util import print_header, run_commands, task + + +@dataclasses.dataclass +class DockerImage: + name: str + architecture: str + version: str + + base_image: str + dockerfile: str + + errors: str + + @property + def tag(self): + architecture = self.architecture + if architecture == "arm32v7": + architecture = "armv7" + return f"daqzilla/{self.name}-{architecture}:{self.version}" + + def build(self, package_file: Path): + + # Sanity checks. + if not package_file.exists(): + message = f"Package {package_file} missing." + if self.errors == "raise": + raise FileNotFoundError(message) + else: + print(f"WARNING: {message}") + return + + # Build Docker image. + print_header(f"Building Docker image {self.tag} for {self.architecture}") + release_date = datetime.utcnow().strftime("%Y-%m-%d") + os.environ["DOCKER_BUILDKIT"] = "1" + commands = [ + f""" + docker build \ + --tag {self.tag} \ + --build-arg BASE_IMAGE={self.base_image} \ + --build-arg PACKAGE_FILE={package_file} \ + --build-arg VERSION={self.version} \ + --build-arg RELEASE_DATE={release_date} \ + --file {self.dockerfile} . + """, + # f"docker tag daqzilla/kotori:{spec.version} daqzilla/kotori:nightly", + # f"docker tag daqzilla/kotori:$(version) daqzilla/kotori:latest" + ] + run_commands(commands) + print() + + def publish(self): + command = f"docker push {self.tag}" + run_commands(command) + + +class DockerImageBuilder: + + recipes: List[DockerImageRecipe] = [ + DockerImageRecipe( + enabled=True, + architecture="amd64", + flavors=["full", "standard"], + base_distribution="buster", + base_image="{architecture}/debian:buster-slim", + platform="linux/amd64", + ), + DockerImageRecipe( + enabled=True, + architecture="arm64v8", + flavors=["standard"], + base_distribution="buster", + base_image="{architecture}/debian:buster-slim", + platform="linux/arm64/v8", + ), + DockerImageRecipe( + enabled=True, + architecture="arm32v7", + flavors=["standard"], + base_distribution="buster", + base_image="{architecture}/debian:buster-slim", + platform="linux/arm/v7", + ), + ] + + manifest_tool_url = "https://github.com/estesp/manifest-tool/releases/download/v1.0.3/manifest-tool-darwin-amd64" + manifest_tool_path = Path("./var/bin/manifest-tool") + + def __init__(self, errors="ignore"): + self.errors = errors + self.manifest_tool = None + self.download_manifest_tool() + + def download_manifest_tool(self): + if not self.manifest_tool_path.exists(): + self.manifest_tool_path.parent.mkdir(parents=True, exist_ok=True) + response = requests.get(self.manifest_tool_url, allow_redirects=True) + open(self.manifest_tool_path, "wb").write(response.content) + self.manifest_tool_path.chmod(0o777) + self.manifest_tool = sh.Command(self.manifest_tool_path) + + @staticmethod + def check_manifest_tool(): + if "DOCKER_USERNAME" not in os.environ or "DOCKER_PASSWORD" not in os.environ: + raise ValueError( + "Unable to build without DOCKER_USERNAME and DOCKER_PASSWORD" + ) + + def run(self, version: str = ""): + + # Sanity checks. + version = version.strip() + if not version: + raise ValueError("Unable to build without version") + + self.check_manifest_tool() + + for recipe in self.recipes: + + if not recipe.enabled: + continue + + for flavor in recipe.flavors: + + spec = PackageSpecification( + distribution=recipe.base_distribution, + architecture=recipe.architecture, + flavor=flavor, + version=version, + ) + + docker_image = DockerImage( + name=spec.name, + architecture=recipe.architecture, + version=version, + base_image=recipe.base_image.format( + architecture=recipe.architecture + ), + dockerfile="packaging/dockerfiles/docker-image.dockerfile", + errors=self.errors, + ) + + # Compute path to `.deb` file. + package_name = spec.deb_name() + package_file = Path("dist") / package_name + + # Build Docker image. + docker_image.build(package_file=package_file) + + # Publish Docker image. + docker_image.publish() + + # Publish manifest needed for multi-architecture images. + self.manifest(basename="kotori", platforms=self.platforms, version=version) + self.manifest( + basename="kotori-standard", platforms=self.platforms, version=version + ) + + @property + def platforms(self): + platforms = set() + for recipe in self.recipes: + platforms.add(recipe.platform) + return list(platforms) + + def manifest( + self, basename: str, platforms: List[str], version: str, tag: str = None + ): + """ + Create multi-architecture Docker images. + + - https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/ + - https://docs.docker.com/docker-for-mac/multi-arch/ + - https://developer.ibm.com/tutorials/createmulti-architecture-docker-images/ + - https://github.com/estesp/manifest-tool + """ + + self.check_manifest_tool() + + print_header( + f"Creating Docker manifest for {basename} on platforms {platforms}" + ) + + # Sanity checks. + if self.manifest_tool is None: + raise FileNotFoundError( + f"Docker manifest-tool not found at {self.manifest_tool_path}" + ) + + if tag is None: + tag = version + + print( + self.manifest_tool( + "--username", os.environ["DOCKER_USERNAME"], + "--password", os.environ["DOCKER_PASSWORD"], + "push", + "from-args", + "--ignore-missing", + "--platforms", ",".join(platforms), + "--template", f"daqzilla/{basename}-ARCHVARIANT:{version}", + "--target", f"daqzilla/{basename}:{tag}", + ) + ) + + def link(self, version: str, tag: str): + + self.check_manifest_tool() + + # Publish manifest for specific tag (e.g. "nightly", "latest"). + self.manifest( + basename="kotori", platforms=self.platforms, version=version, tag=tag + ) + self.manifest( + basename="kotori-standard", + platforms=self.platforms, + version=version, + tag=tag, + ) + + def qa(self, tag: str = None): + for recipe in self.recipes: + + if not recipe.enabled: + continue + + for flavor in recipe.flavors: + + # TODO: This is currently only needed to resolve the "name" from the "flavor". Refactor it! + spec = PackageSpecification( + distribution=recipe.base_distribution, + architecture=recipe.architecture, + flavor=flavor, + version="n/a", + ) + + image_name = f"daqzilla/{spec.name}" + if tag: + image_name += f":{tag}" + + print_header( + f"Running QA for Docker image {image_name} on {recipe.platform}" + ) + + command = f"docker run -it --rm --platform={recipe.platform} {image_name} kotori --version" + run_commands(command) + + print() + + +@task +def images(context: Context, version: str = None): + """ + Build all designated images for pushing to Docker Hub (stage three). + """ + DockerImageBuilder().run(version=version) + + +@task +def link(context: Context, version: str = None, tag: str = None): + """ + Link image versions to tags (stage three). + """ + if not version or not tag: + raise ValueError("Linking a repository needs version and tag") + DockerImageBuilder().link(version=version, tag=tag) + + +@task +def qa(context: Context, tag: str = None): + """ + Run basic quality assurance on Docker images (stage four). + """ + if not tag: + tag = None + DockerImageBuilder().qa(tag=tag) diff --git a/tasks/packaging/environment.py b/tasks/packaging/environment.py new file mode 100644 index 00000000..9b74f403 --- /dev/null +++ b/tasks/packaging/environment.py @@ -0,0 +1,85 @@ +from typing import List + +from tasks.packaging.model import DockerBaselineImageRecipe +from tasks.util import print_header, run_commands, task + + +class DockerBaselineImageBuilder: + """ + Build baseline images reflecting stage one of the whole process. + """ + + recipes: List[DockerBaselineImageRecipe] = [ + # Vanilla Debian. + DockerBaselineImageRecipe( + enabled=True, + vendor="Debian", + distributions=[ + "stretch", + "buster", + # "bullseye", + ], + architectures=[ + "amd64", + "arm64v8", + "arm32v7", + ], + image="{architecture}/debian:{distribution}-slim", + ), + # Ubuntu. + DockerBaselineImageRecipe( + enabled=True, + vendor="Ubuntu", + distributions=[ + "bionic", + "focal", + # "hirsute", + ], + architectures=[ + "amd64", + ], + image="{architecture}/ubuntu:{distribution}", + ), + ] + + docker_image_version = "0.10.0" + + def run(self): + """ + Invoke the image building for all enabled recipes. + """ + + for recipe in self.recipes: + + # Skip recipes not enabled. + if not recipe.enabled: + continue + + # Build all distributions. + for distribution in recipe.distributions: + + # Build all architectures per distribution. + for architecture in recipe.architectures: + print_header( + f'Building baseline image for {recipe.vendor} "{distribution}" on {architecture}' + ) + image = recipe.image.format(**locals()) + commands = [ + f""" + docker build \ + --tag daq-tools/{distribution}-{architecture}-baseline:{self.docker_image_version} \ + --tag daq-tools/{distribution}-{architecture}-baseline:latest \ + --build-arg BASE_IMAGE={image} - \ + < packaging/dockerfiles/debian-baseline.dockerfile + """, + ] + run_commands(commands) + print() + + +@task +def baseline_images(context): + """ + Build all Docker baseline images (stage one). + """ + DockerBaselineImageBuilder().run() diff --git a/tasks/packaging/model.py b/tasks/packaging/model.py new file mode 100644 index 00000000..4e3fefb5 --- /dev/null +++ b/tasks/packaging/model.py @@ -0,0 +1,134 @@ +import dataclasses +from typing import ClassVar, List + + +@dataclasses.dataclass +class DockerBaselineImageRecipe: + """ + Data structure for holding information about a recipe for building baseline + Docker images. Those images will be used for building the actual operating + system distribution packages. + """ + + # Whether this recipe is enabled. + enabled: bool + + # The operating system vendor, e.g. Debian, Ubuntu. + vendor: str + + # List of operating system distributions, e.g. stretch, buster. + distributions: List[str] + + # List of architectures, e.g. amd64, arm64v8, arm32v7. + architectures: List[str] + + # Docker image to use. + image: str + + +@dataclasses.dataclass +class PackageSpecification: + """ + Data structure for holding information about individual distribution + packages. + """ + + # Package flavor, e.g. minimal, standard, full. + flavor: str + + # Package version. + version: str + + # Distribution name, e.g. stretch, buster. + distribution: str + + # Architecture, e.g. amd64, arm64v8, arm32v7. + architecture: str + + # Computed package name, taking `flavor` into account. + name: str = None + + # List of Python `extra` labels, taking `flavor` into account. + features: str = None + + # Map architecture labels to Debian distribution package architecture name. + debian_architecture_map: ClassVar = { + "arm64v8": "arm64", + "arm32v7": "armhf", + } + + def __post_init__(self): + self.resolve() + self.validate() + + def validate(self): + """ + Sanity checks. All designated specification fields must be set and not + be empty. + """ + for field in dataclasses.fields(self): + value = getattr(self, field.name) + if value is None: + raise ValueError( + f'Package specification field "{field.name}" is required.' + ) + + def resolve(self): + """ + Resolve the designated package flavor to a list of Python package + `extra` labels. + """ + if self.flavor == "minimal": + self.name = "kotori-minimal" + self.features = "daq" + elif self.flavor == "standard": + self.name = "kotori-standard" + self.features = "daq,daq_geospatial,export" + elif self.flavor == "full": + self.name = "kotori" + self.features = "daq,daq_geospatial,export,plotting,firmware,scientific" + else: + raise ValueError("Unknown package flavor") + + @property + def debian_architecture(self): + """ + Resolve the architecture label to a Debian package architecture suffix. + """ + return self.debian_architecture_map.get(self.architecture, self.architecture) + + def deb_name(self): + """ + Compute the full name of the Debian `.deb` package. + """ + return f"{self.name}_{self.version}-1~{self.distribution}_{self.debian_architecture}.deb" + + +@dataclasses.dataclass +class PackageRecipe: + """ + Data structure for holding information about a recipe for building + distribution packages. + """ + + # Whether this recipe is enabled. + enabled: bool + + # Architecture, e.g. amd64, arm64v8, arm32v7. + architecture: str + + # List of operating system distributions, e.g. stretch, buster. + distributions: List[str] + + # List of package flavors, e.g. minimal, standard, full. + flavors: List[str] + + +@dataclasses.dataclass +class DockerImageRecipe: + enabled: bool + architecture: str + flavors: List[str] + base_distribution: str + base_image: str + platform: str diff --git a/tasks/packaging/ospackage.py b/tasks/packaging/ospackage.py new file mode 100644 index 00000000..11c37714 --- /dev/null +++ b/tasks/packaging/ospackage.py @@ -0,0 +1,153 @@ +import dataclasses +import json +from pathlib import Path + +from invoke import Context + +from tasks.packaging.model import PackageRecipe, PackageSpecification +from tasks.util import print_header, run_commands, task + +map() + +class PackageBuilder: + """ + Build distribution packages reflecting stage two of the whole process. + """ + + recipes = [ + PackageRecipe( + enabled=True, + architecture="amd64", + distributions=[ + "stretch", + "buster", + "bionic", + "focal", + ], + flavors=[ + "full", + "standard", + ], + ), + PackageRecipe( + enabled=True, + architecture="arm64v8", + distributions=[ + "stretch", # ImportError: cannot import name '_BACKCOMPAT_MAGIC_NUMBER' + "buster", + ], + flavors=["standard"], + ), + PackageRecipe( + enabled=True, + architecture="arm32v7", + distributions=[ + "stretch", # ImportError: cannot import name '_BACKCOMPAT_MAGIC_NUMBER' + "buster", + ], + flavors=["standard"], + ), + ] + + def run(self, version=""): + """ + Invoke the package building for all enabled recipes. + """ + + # Sanity checks. The version must be set and not empty. + version = version.strip() + if not version: + raise ValueError("Unable to build without version") + + for recipe in self.recipes: + + # Skip recipes not enabled. + if not recipe.enabled: + continue + + architecture = recipe.architecture + for distribution in recipe.distributions: + for flavor in recipe.flavors: + spec = PackageSpecification( + distribution=distribution, + architecture=architecture, + flavor=flavor, + version=version, + ) + self.deb(spec) + + @staticmethod + def build_container_name(spec): + return f"daq-tools/kotori-build-{spec.distribution}-{spec.architecture}:{spec.version}" + + def deb(self, spec: PackageSpecification): + + # Compute package name and path to `.deb` file. + package_name = spec.deb_name() + package_file = Path("dist") / package_name + + # Sanity checks. + if package_file.exists(): + print(f"Package {package_file} already exists, skipping.") + return + + print_header( + f"Building package {spec.name} for {spec.distribution} on {spec.architecture}" + ) + print(json.dumps(dataclasses.asdict(spec), indent=4)) + + # Build Linux distribution package within Docker container. + command = f""" + docker build \ + --tag {self.build_container_name(spec)} \ + --build-arg BASE_IMAGE=daq-tools/{spec.distribution}-{spec.architecture}-baseline:latest \ + --build-arg NAME={spec.name} \ + --build-arg FEATURES={spec.features} \ + --build-arg VERSION={spec.version} \ + --build-arg DISTRIBUTION={spec.distribution} \ + --build-arg ARCHITECTURE={spec.architecture} \ + --file packaging/dockerfiles/debian-package.dockerfile . + """ + run_commands(command) + + # Extract package from Docker container. + tempname = "pkg_extract__" + commands = [ + f"docker container rm --force {tempname}", + f"docker container create --name={tempname} {self.build_container_name(spec)}", + f"docker container cp {tempname}:/dist/{package_name} ./dist/", + f"docker container rm --force {tempname}", + ] + run_commands(commands) + print() + + +@task +def deb( + context: Context, + distribution: str = None, + architecture: str = None, + flavor: str = None, + version: str = None, +): + """ + Build an individual Debian package. + """ + spec = PackageSpecification( + distribution=distribution, + architecture=architecture, + flavor=flavor, + version=version, + ) + PackageBuilder().deb(spec) + + +@task +def run(context, version): + """ + Build all operating system distribution packages (stage two). + """ + print() + print_header(f"Building packages for Kotori version {version}", "=") + print() + PackageBuilder().run(version=version) diff --git a/tasks/util.py b/tasks/util.py new file mode 100644 index 00000000..44a960b0 --- /dev/null +++ b/tasks/util.py @@ -0,0 +1,31 @@ +# Needed for typed args. +# https://github.com/pyinvoke/invoke/issues/357#issuecomment-688596802 +import os +from typing import Any, Callable, TypeVar + +import invoke + +F = TypeVar("F", bound=Callable[..., Any]) + + +def task(f: F) -> F: + f.__annotations__ = {} + return invoke.task(f) + + +def run_commands(commands): + if isinstance(commands, str): + commands = [commands] + for command in commands: + command = command.strip() + print(command) + exitcode = os.system(command) + if exitcode != 0: + raise SystemExit("Command failed") + + +def print_header(label, char="-"): + length = max(len(label), 42) + print(char * length) + print(label.center(length)) + print(char * length) diff --git a/test/util.py b/test/util.py index 9ab6fdae..fcf8acdd 100644 --- a/test/util.py +++ b/test/util.py @@ -153,9 +153,9 @@ def mqtt_sensor(topic, payload): # https://stackoverflow.com/questions/24319662 if os.environ.get("CI"): if sys.platform == "linux": - mosquitto_pub = "docker run --rm --network=host eclipse-mosquitto:1.6.12 mosquitto_pub -h localhost" + mosquitto_pub = "docker run --rm --network=host eclipse-mosquitto:1.6 mosquitto_pub -h localhost" elif sys.platform == "darwin": - mosquitto_pub = "docker run --rm eclipse-mosquitto:1.6.12 mosquitto_pub -h host.docker.internal" + mosquitto_pub = "docker run --rm eclipse-mosquitto:1.6 mosquitto_pub -h host.docker.internal" else: raise NotImplementedError("Invoking 'mosquitto_pub' through Docker on '{}' not supported yet".format(sys.platform)) else: