From 53ca0481bbffcb2eeda5bcd3719df88c4c82f78e Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:32 +0200 Subject: [PATCH 01/24] FIX: hash only `.py` files under the `src` directory Prevents untracked files from being part of the cache key --- .github/workflows/ci-docs.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 12fb8ce3..93751276 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -102,27 +102,27 @@ jobs: uses: actions/cache@v3 with: key: | - sympy-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}}-${{hashFiles('src/**')}}-${{hashFiles('docs/**.ipynb')}} + sympy-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}}-${{hashFiles('src/**.py')}}-${{hashFiles('docs/**.ipynb')}} restore-keys: | - sympy-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}}-${{hashFiles('src/**')}} + sympy-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}}-${{hashFiles('src/**.py')}} sympy-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}} path: ~/.sympy-cache*/ - name: Fetch Jupyter cache uses: actions/cache@v3 with: key: | - jupyter-cache-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**')}}-${{hashFiles('docs/**')}} + jupyter-cache-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**.py')}}-${{hashFiles('docs/**')}} restore-keys: | - jupyter-cache-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**')}} + jupyter-cache-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**.py')}} path: | ./docs/_build/.jupyter_cache - name: Fetch output files uses: actions/cache@v3 with: key: | - output-files-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**')}}-${{hashFiles('docs/**')}} + output-files-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**.py')}}-${{hashFiles('docs/**')}} restore-keys: | - output-files-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**')}} + output-files-${{hashFiles('.constraints/py3.*.txt', 'data/**', 'setup.cfg', 'src/**.py')}} output-files path: | ./docs/_images From 8a3bbf1273e8aca517aefda8a31cd150af385f35 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:36 +0200 Subject: [PATCH 02/24] FIX: correctly hash `.jl` files There were no matched files for the previous pattern, see https://github.com/ComPWA/polarimetry/actions/runs/3962250099/jobs/6788669966 --- .github/workflows/ci-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 93751276..6126dacc 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -78,7 +78,7 @@ jobs: uses: actions/cache@v3 with: key: | - julia-${{hashFiles('julia/Manifest.toml')}}-${{hashFiles('**.jl')}} + julia-${{hashFiles('julia/Manifest.toml')}}-${{hashFiles('./**.jl')}} restore-keys: | julia-${{hashFiles('julia/Manifest.toml')}} julia From 918bbcf1431d806ce02b7c5efbc1f7bbcddc973d Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:37 +0200 Subject: [PATCH 03/24] DX: run slow documentation job on PRs --- .github/workflows/ci-docs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 6126dacc..df6f6550 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -63,7 +63,6 @@ jobs: documentation: name: Run notebooks and build documentation - if: github.event_name != 'pull_request' runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 From edff44766fa165a4c8e655cd2a82bf70bbb6bad4 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:38 +0200 Subject: [PATCH 04/24] DX: run fast doc jobs in Docker containers Removes the need to run `apt install` Dockerfile: ```Dockerfile FROM ubuntu:22.04 RUN apt-get -y update \# https://serverfault.com/a/992421 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata RUN apt-get -y install inkscape latexmk make texlive-fonts-extra texlive-xetex xindy RUN apt-get clean RUN rm -rf /var/lib/apt/lists/* ``` and uploaded it to Docker Hub: ```shell sudo docker build -t compwa/polarimetry-fast-pdf:v1 . sudo docker tag compwa/polarimetry-fast-pdf:v1 compwa/polarimetry-fast-pdf:v1 sudo docker push compwa/polarimetry-fast-pdf:v1 ``` --- .github/workflows/ci-docs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index df6f6550..06301ba1 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -14,6 +14,7 @@ jobs: fast-html: if: inputs.specific-pip-packages == '' name: Build HTML (fast) + container: compwa/polarimetry-fast-html:v1 runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -32,6 +33,7 @@ jobs: fast-pdf: if: inputs.specific-pip-packages == '' name: Build PDF (fast) + container: compwa/polarimetry-fast-pdf:v1 runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -40,7 +42,6 @@ jobs: additional-packages: tox extras: doc python-version: ${{env.PYTHON_VERSION}} - - run: sudo apt install -y inkscape latexmk make texlive-fonts-extra texlive-xetex xindy - run: tox -e pdf - uses: actions/upload-artifact@v3 if: ${{ always() }} From 959133c1e3753f7501c3c7c771ecb2c9b9f1ef87 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:39 +0200 Subject: [PATCH 05/24] FIX: set Docker container through `container.image` --- .github/workflows/ci-docs.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 06301ba1..02ac3b46 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -14,7 +14,8 @@ jobs: fast-html: if: inputs.specific-pip-packages == '' name: Build HTML (fast) - container: compwa/polarimetry-fast-html:v1 + container: + image: compwa/polarimetry-fast-html:v1 runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -33,7 +34,8 @@ jobs: fast-pdf: if: inputs.specific-pip-packages == '' name: Build PDF (fast) - container: compwa/polarimetry-fast-pdf:v1 + container: + image: compwa/polarimetry-fast-pdf:v1 runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 From 82358ef604aeb35951b3ad6927cdd7ca5f51a3db Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:39 +0200 Subject: [PATCH 06/24] DX: speed up slow doc job with `compwa/polarimetry-documentation:v1` container --- .github/workflows/ci-docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 02ac3b46..eb5dc53e 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -67,6 +67,8 @@ jobs: documentation: name: Run notebooks and build documentation runs-on: ubuntu-22.04 + container: + image: compwa/polarimetry-documentation:v1 steps: - uses: actions/checkout@v3 - uses: ComPWA/actions/pip-install@v1 From 447613ffda369d35cd1fe880d459bd50a3be281c Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:40 +0200 Subject: [PATCH 07/24] FIX: do not setup Python when using `container.image` --- .github/workflows/ci-docs.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index eb5dc53e..de308398 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -39,11 +39,14 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - - uses: ComPWA/actions/pip-install@v1 + - uses: actions/cache@v3 with: - additional-packages: tox - extras: doc - python-version: ${{env.PYTHON_VERSION}} + key: | + pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}}-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}} + restore-keys: | + pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}} + path: ~/.cache/pip/ + - run: pip install .[doc] tox -c .constraints/py${{env.PYTHON_VERSION}}.txt - run: tox -e pdf - uses: actions/upload-artifact@v3 if: ${{ always() }} @@ -71,12 +74,14 @@ jobs: image: compwa/polarimetry-documentation:v1 steps: - uses: actions/checkout@v3 - - uses: ComPWA/actions/pip-install@v1 + - uses: actions/cache@v3 with: - additional-packages: tox - extras: doc - python-version: ${{env.PYTHON_VERSION}} - specific-packages: ${{ inputs.specific-pip-packages }} + key: | + pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}}-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}} + restore-keys: | + pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}} + path: ~/.cache/pip/ + - run: pip install .[doc] tox -c .constraints/py${{env.PYTHON_VERSION}}.txt - run: sudo apt-get -y install cm-super dvipng inkscape latexmk texlive-fonts-extra texlive-latex-extra texlive-xetex xindy - name: Fetch Julia cache uses: actions/cache@v3 From 92814b57e5e46a880ccebaf15c72cff89fa2fd8c Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:41 +0200 Subject: [PATCH 08/24] FIX: remove `apt-get` install statement --- .github/workflows/ci-docs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index de308398..23e99d55 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -82,7 +82,6 @@ jobs: pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}} path: ~/.cache/pip/ - run: pip install .[doc] tox -c .constraints/py${{env.PYTHON_VERSION}}.txt - - run: sudo apt-get -y install cm-super dvipng inkscape latexmk texlive-fonts-extra texlive-latex-extra texlive-xetex xindy - name: Fetch Julia cache uses: actions/cache@v3 with: From d613511d084aafe462552c56c812acf64e168104 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:42 +0200 Subject: [PATCH 09/24] MAINT: add `Dockerfile`s --- .github/workflows/docker/Dockerfile.documentation | 11 +++++++++++ .github/workflows/docker/Dockerfile.fast-pdf | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 .github/workflows/docker/Dockerfile.documentation create mode 100644 .github/workflows/docker/Dockerfile.fast-pdf diff --git a/.github/workflows/docker/Dockerfile.documentation b/.github/workflows/docker/Dockerfile.documentation new file mode 100644 index 00000000..6865eea8 --- /dev/null +++ b/.github/workflows/docker/Dockerfile.documentation @@ -0,0 +1,11 @@ +FROM python:3.8 +FROM julia:1.8-alpine +RUN apt-get -y update + +# https://serverfault.com/a/992421 +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata +RUN apt-get -y install cm-super dvipng inkscape latexmk texlive-fonts-extra texlive-latex-extra texlive-xetex xindy + +RUN apt-get autoclean -y +RUN apt-get autoremove -y +RUN rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/docker/Dockerfile.fast-pdf b/.github/workflows/docker/Dockerfile.fast-pdf new file mode 100644 index 00000000..bf5a0861 --- /dev/null +++ b/.github/workflows/docker/Dockerfile.fast-pdf @@ -0,0 +1,10 @@ +FROM python:3.8-alpine +RUN apt-get -y update + +# https://serverfault.com/a/992421 +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata +RUN apt-get -y install inkscape latexmk make texlive-fonts-extra texlive-xetex xindy + +RUN apt-get autoclean -y +RUN apt-get autoremove -y +RUN rm -rf /var/lib/apt/lists/* From 26faf70ad7a19a4c3b5ad32009065d99c076bade Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:10:40 +0200 Subject: [PATCH 10/24] DX: add Docker VS Code extension --- .vscode/extensions.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index be043bf6..5173cc45 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,12 +1,13 @@ { "recommendations": [ - "ExecutableBookProject.myst-highlight", "bungcip.better-toml", "eamodio.gitlens", "editorconfig.editorconfig", "esbenp.prettier-vscode", + "executablebookproject.myst-highlight", "github.vscode-pull-request-github", "julialang.language-julia", + "ms-azuretools.vscode-docker", "ms-python.python", "ms-python.vscode-pylance", "ms-toolsai.vscode-jupyter-cell-tags", From 701e73459e27114a22776d96c30f15b17e5dd856 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:13:15 +0200 Subject: [PATCH 11/24] FIX: avoid Alpine in Docker containers --- .cspell.json | 1 + .github/workflows/ci-docs.yml | 2 +- .../workflows/docker/Dockerfile.documentation | 20 ++++++++++++------- .github/workflows/docker/Dockerfile.fast-pdf | 17 ++++++++++------ 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.cspell.json b/.cspell.json index b857e304..6572597b 100644 --- a/.cspell.json +++ b/.cspell.json @@ -51,6 +51,7 @@ ], "ignorePaths": [ "**/.cspell.json", + "**/Dockerfile*", "*.bib", ".constraints/*.txt", ".cspell/*.txt", diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 23e99d55..43d03d4a 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -1,7 +1,7 @@ env: JULIA_CI: "true" PYTHONHASHSEED: "0" - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" on: workflow_call: diff --git a/.github/workflows/docker/Dockerfile.documentation b/.github/workflows/docker/Dockerfile.documentation index 6865eea8..e006baeb 100644 --- a/.github/workflows/docker/Dockerfile.documentation +++ b/.github/workflows/docker/Dockerfile.documentation @@ -1,11 +1,17 @@ -FROM python:3.8 -FROM julia:1.8-alpine +FROM julia:1.8 RUN apt-get -y update # https://serverfault.com/a/992421 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt-get -y install cm-super dvipng inkscape latexmk texlive-fonts-extra texlive-latex-extra texlive-xetex xindy - -RUN apt-get autoclean -y -RUN apt-get autoremove -y -RUN rm -rf /var/lib/apt/lists/* +RUN apt-get -y install \ + cm-super \ + dvipng \ + inkscape \ + latexmk \ + texlive-fonts-extra \ + texlive-latex-extra \ + texlive-xetex \ + xindy \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/docker/Dockerfile.fast-pdf b/.github/workflows/docker/Dockerfile.fast-pdf index bf5a0861..6cec56af 100644 --- a/.github/workflows/docker/Dockerfile.fast-pdf +++ b/.github/workflows/docker/Dockerfile.fast-pdf @@ -1,10 +1,15 @@ -FROM python:3.8-alpine +FROM python:3.9 RUN apt-get -y update # https://serverfault.com/a/992421 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt-get -y install inkscape latexmk make texlive-fonts-extra texlive-xetex xindy - -RUN apt-get autoclean -y -RUN apt-get autoremove -y -RUN rm -rf /var/lib/apt/lists/* +RUN apt-get -y install \ + inkscape \ + latexmk \ + make \ + texlive-fonts-extra \ + texlive-xetex \ + xindy \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* From 04033453ae24d2b811c0b59f2bec5067545aaa71 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:20:12 +0200 Subject: [PATCH 12/24] DX: automatically build Docker images --- .github/workflows/docker.yaml | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/docker.yaml diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 00000000..5502cc90 --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,38 @@ +name: Update Docker images + +on: + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_BASE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + permissions: + contents: read + packages: write + runs-on: ubuntu-22.04 + strategy: + matrix: + job-name: + - documentation + - fast-pdf + steps: + - uses: actions/checkout@v3 + - uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE_NAME }}-${{ matrix.job-name }} + - uses: docker/build-push-action@v4 + with: + context: . + file: .github/workflows/docker/Dockerfile.${{ matrix.job-name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From e600ed22f3c7be3fde454def534b77cc9e861f19 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:25:15 +0200 Subject: [PATCH 13/24] MAINT: push Docker container with SHA This allows pinning to a specific Docker container --- .github/workflows/docker.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 5502cc90..9b46334f 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -29,6 +29,9 @@ jobs: uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE_NAME }}-${{ matrix.job-name }} + tags: | + type=sha + type=sha,format=long - uses: docker/build-push-action@v4 with: context: . From a25a80affc825d0d5bd5a9e3759039ff839c8aa4 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:25:45 +0200 Subject: [PATCH 14/24] TEMP: build Docker containers on push to `docker-images` branch --- .github/workflows/docker.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 9b46334f..be16069b 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,6 +1,9 @@ name: Update Docker images on: + push: + branches: + - docker-images workflow_dispatch: env: From 35a938d260adeeb86eaf291ef614ba0c4f8a9d27 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:29:40 +0200 Subject: [PATCH 15/24] DX: add name to Docker container build job --- .github/workflows/docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index be16069b..9fbab5fb 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -12,6 +12,7 @@ env: jobs: build-and-push-image: + name: Build Docker image permissions: contents: read packages: write From ff58f869ebb7ac45052f168a7b4d3f8d24a918b3 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:39:09 +0200 Subject: [PATCH 16/24] MAINT: fine-grain Docker image tag --- .github/workflows/docker.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 9fbab5fb..434b842c 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -34,8 +34,14 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE_NAME }}-${{ matrix.job-name }} tags: | + type=schedule + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=branch + type=ref,event=pr + type=edge,branch=main type=sha - type=sha,format=long - uses: docker/build-push-action@v4 with: context: . From fabf38c260f0d8734dfa322dffa85c19a6d1bf0b Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:46:35 +0200 Subject: [PATCH 17/24] DX: speed up workflow with Docker containers --- .github/workflows/ci-docs.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 43d03d4a..8a9c4731 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -14,8 +14,6 @@ jobs: fast-html: if: inputs.specific-pip-packages == '' name: Build HTML (fast) - container: - image: compwa/polarimetry-fast-html:v1 runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -35,7 +33,10 @@ jobs: if: inputs.specific-pip-packages == '' name: Build PDF (fast) container: - image: compwa/polarimetry-fast-pdf:v1 + image: ghcr.io/compwa/polarimetry-fast-pdf:sha-a25a80a + credentials: + username: ${{ github.actor }} + password: ${{ secrets.github_token }} runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -71,7 +72,10 @@ jobs: name: Run notebooks and build documentation runs-on: ubuntu-22.04 container: - image: compwa/polarimetry-documentation:v1 + image: ghcr.io/compwa/polarimetry-documentation:sha-a25a80a + credentials: + username: ${{ github.actor }} + password: ${{ secrets.github_token }} steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 @@ -91,19 +95,9 @@ jobs: julia-${{hashFiles('julia/Manifest.toml')}} julia path: | - ./julia-*/ ~/.julia/ - - name: Install Julia + - name: Initialize Julia environment run: | - version=$(sed -n '3p' julia/Manifest.toml) - version=${version:17:-1} - major_version=${version:0:-2} - filename=julia-${version}-linux-x86_64.tar.gz - if [ ! -d julia-${version} ]; then - wget -q https://julialang-s3.julialang.org/bin/linux/x64/${major_version}/${filename} - tar xzf ${filename} - fi - sudo ln -s $(pwd)/julia-${version}/bin/julia /usr/local/bin/julia julia --version julia --project=./julia -e 'import Pkg; Pkg.instantiate()' - name: Fetch SymPy cache From 0c0224379b7065863931df7c4a208af94263d2d9 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:57:07 +0200 Subject: [PATCH 18/24] MAINT: install packages with `ComPWA/actions/pip-install` --- .github/workflows/ci-docs.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 8a9c4731..33f34cec 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -40,14 +40,11 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: ComPWA/actions/pip-install@v1 with: - key: | - pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}}-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}} - restore-keys: | - pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}} - path: ~/.cache/pip/ - - run: pip install .[doc] tox -c .constraints/py${{env.PYTHON_VERSION}}.txt + additional-packages: tox + extras: doc + python-version: ${{env.PYTHON_VERSION}} - run: tox -e pdf - uses: actions/upload-artifact@v3 if: ${{ always() }} @@ -78,14 +75,12 @@ jobs: password: ${{ secrets.github_token }} steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: ComPWA/actions/pip-install@v1 with: - key: | - pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}}-${{hashFiles('.constraints/py3.*.txt', 'setup.cfg')}} - restore-keys: | - pip-doc-${{runner.os}}-py${{env.PYTHON_VERSION}} - path: ~/.cache/pip/ - - run: pip install .[doc] tox -c .constraints/py${{env.PYTHON_VERSION}}.txt + additional-packages: tox + extras: doc + python-version: ${{env.PYTHON_VERSION}} + specific-packages: ${{ inputs.specific-pip-packages }} - name: Fetch Julia cache uses: actions/cache@v3 with: From dd22a9c4b4226b2e819d68c4c3a8be0ea0cf5886 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 14:04:46 +0200 Subject: [PATCH 19/24] MAINT: update docker SHA --- .github/workflows/ci-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 33f34cec..07b98ba2 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -33,7 +33,7 @@ jobs: if: inputs.specific-pip-packages == '' name: Build PDF (fast) container: - image: ghcr.io/compwa/polarimetry-fast-pdf:sha-a25a80a + image: ghcr.io/compwa/polarimetry-fast-pdf:sha-fabf38c credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -69,7 +69,7 @@ jobs: name: Run notebooks and build documentation runs-on: ubuntu-22.04 container: - image: ghcr.io/compwa/polarimetry-documentation:sha-a25a80a + image: ghcr.io/compwa/polarimetry-documentation:sha-fabf38c credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} From 7e5062449bca766af830707adc610ffd301175c4 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:37:43 +0200 Subject: [PATCH 20/24] FIX: install Python explicitly in Docker --- .github/workflows/ci-docs.yml | 4 ++-- .github/workflows/docker/Dockerfile.documentation | 2 ++ .github/workflows/docker/Dockerfile.fast-pdf | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 07b98ba2..2e752399 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -33,7 +33,7 @@ jobs: if: inputs.specific-pip-packages == '' name: Build PDF (fast) container: - image: ghcr.io/compwa/polarimetry-fast-pdf:sha-fabf38c + image: ghcr.io/compwa/polarimetry-fast-pdf:sha-dd22a9c credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -69,7 +69,7 @@ jobs: name: Run notebooks and build documentation runs-on: ubuntu-22.04 container: - image: ghcr.io/compwa/polarimetry-documentation:sha-fabf38c + image: ghcr.io/compwa/polarimetry-documentation:sha-dd22a9c credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} diff --git a/.github/workflows/docker/Dockerfile.documentation b/.github/workflows/docker/Dockerfile.documentation index e006baeb..dc5ee7ad 100644 --- a/.github/workflows/docker/Dockerfile.documentation +++ b/.github/workflows/docker/Dockerfile.documentation @@ -8,6 +8,8 @@ RUN apt-get -y install \ dvipng \ inkscape \ latexmk \ + python3 \ + python3-pip \ texlive-fonts-extra \ texlive-latex-extra \ texlive-xetex \ diff --git a/.github/workflows/docker/Dockerfile.fast-pdf b/.github/workflows/docker/Dockerfile.fast-pdf index 6cec56af..6d70ff5d 100644 --- a/.github/workflows/docker/Dockerfile.fast-pdf +++ b/.github/workflows/docker/Dockerfile.fast-pdf @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM ubuntu:22.04 RUN apt-get -y update # https://serverfault.com/a/992421 @@ -7,6 +7,8 @@ RUN apt-get -y install \ inkscape \ latexmk \ make \ + python3 \ + python3-pip \ texlive-fonts-extra \ texlive-xetex \ xindy \ From 58b759694f61a05b9d4a036c1d7e829328c27e08 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:58:50 +0200 Subject: [PATCH 21/24] MAINT: run Docker with eforced Python version (7e50624) --- .github/workflows/ci-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 2e752399..50407e3f 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -33,7 +33,7 @@ jobs: if: inputs.specific-pip-packages == '' name: Build PDF (fast) container: - image: ghcr.io/compwa/polarimetry-fast-pdf:sha-dd22a9c + image: ghcr.io/compwa/polarimetry-fast-pdf:sha-7e50624 credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -69,7 +69,7 @@ jobs: name: Run notebooks and build documentation runs-on: ubuntu-22.04 container: - image: ghcr.io/compwa/polarimetry-documentation:sha-dd22a9c + image: ghcr.io/compwa/polarimetry-documentation:sha-7e50624 credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} From 031bc430b9584267801e3cd1cb1b63f5ef98b764 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:24:37 +0200 Subject: [PATCH 22/24] MAINT: move autopush-docker to ComPWA/polarimetry#299 --- .cspell.json | 1 - .github/workflows/docker.yaml | 51 ------------------- .../workflows/docker/Dockerfile.documentation | 19 ------- .github/workflows/docker/Dockerfile.fast-pdf | 17 ------- .vscode/extensions.json | 3 +- 5 files changed, 1 insertion(+), 90 deletions(-) delete mode 100644 .github/workflows/docker.yaml delete mode 100644 .github/workflows/docker/Dockerfile.documentation delete mode 100644 .github/workflows/docker/Dockerfile.fast-pdf diff --git a/.cspell.json b/.cspell.json index 6572597b..b857e304 100644 --- a/.cspell.json +++ b/.cspell.json @@ -51,7 +51,6 @@ ], "ignorePaths": [ "**/.cspell.json", - "**/Dockerfile*", "*.bib", ".constraints/*.txt", ".cspell/*.txt", diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml deleted file mode 100644 index 434b842c..00000000 --- a/.github/workflows/docker.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: Update Docker images - -on: - push: - branches: - - docker-images - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_BASE_NAME: ${{ github.repository }} - -jobs: - build-and-push-image: - name: Build Docker image - permissions: - contents: read - packages: write - runs-on: ubuntu-22.04 - strategy: - matrix: - job-name: - - documentation - - fast-pdf - steps: - - uses: actions/checkout@v3 - - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE_NAME }}-${{ matrix.job-name }} - tags: | - type=schedule - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=ref,event=branch - type=ref,event=pr - type=edge,branch=main - type=sha - - uses: docker/build-push-action@v4 - with: - context: . - file: .github/workflows/docker/Dockerfile.${{ matrix.job-name }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker/Dockerfile.documentation b/.github/workflows/docker/Dockerfile.documentation deleted file mode 100644 index dc5ee7ad..00000000 --- a/.github/workflows/docker/Dockerfile.documentation +++ /dev/null @@ -1,19 +0,0 @@ -FROM julia:1.8 -RUN apt-get -y update - -# https://serverfault.com/a/992421 -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt-get -y install \ - cm-super \ - dvipng \ - inkscape \ - latexmk \ - python3 \ - python3-pip \ - texlive-fonts-extra \ - texlive-latex-extra \ - texlive-xetex \ - xindy \ - && apt-get autoclean -y \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/docker/Dockerfile.fast-pdf b/.github/workflows/docker/Dockerfile.fast-pdf deleted file mode 100644 index 6d70ff5d..00000000 --- a/.github/workflows/docker/Dockerfile.fast-pdf +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:22.04 -RUN apt-get -y update - -# https://serverfault.com/a/992421 -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt-get -y install \ - inkscape \ - latexmk \ - make \ - python3 \ - python3-pip \ - texlive-fonts-extra \ - texlive-xetex \ - xindy \ - && apt-get autoclean -y \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 5173cc45..be043bf6 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,13 +1,12 @@ { "recommendations": [ + "ExecutableBookProject.myst-highlight", "bungcip.better-toml", "eamodio.gitlens", "editorconfig.editorconfig", "esbenp.prettier-vscode", - "executablebookproject.myst-highlight", "github.vscode-pull-request-github", "julialang.language-julia", - "ms-azuretools.vscode-docker", "ms-python.python", "ms-python.vscode-pylance", "ms-toolsai.vscode-jupyter-cell-tags", From 71ee68fca8585e27d54e7fce6d6577c85c98e133 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:26:29 +0200 Subject: [PATCH 23/24] FIX: do not install Python through `actions/setup-python` --- .github/workflows/ci-docs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 50407e3f..9a147517 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -44,7 +44,6 @@ jobs: with: additional-packages: tox extras: doc - python-version: ${{env.PYTHON_VERSION}} - run: tox -e pdf - uses: actions/upload-artifact@v3 if: ${{ always() }} @@ -79,7 +78,6 @@ jobs: with: additional-packages: tox extras: doc - python-version: ${{env.PYTHON_VERSION}} specific-packages: ${{ inputs.specific-pip-packages }} - name: Fetch Julia cache uses: actions/cache@v3 From d33ace1dc3ce71530eb5d66fae8417c34b100e7c Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:27:13 +0200 Subject: [PATCH 24/24] MAINT: update Docker SHA --- .github/workflows/ci-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 9a147517..eeac6dcf 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -33,7 +33,7 @@ jobs: if: inputs.specific-pip-packages == '' name: Build PDF (fast) container: - image: ghcr.io/compwa/polarimetry-fast-pdf:sha-7e50624 + image: ghcr.io/compwa/polarimetry-fast-pdf:sha-37b24e6 credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -68,7 +68,7 @@ jobs: name: Run notebooks and build documentation runs-on: ubuntu-22.04 container: - image: ghcr.io/compwa/polarimetry-documentation:sha-7e50624 + image: ghcr.io/compwa/polarimetry-documentation:sha-37b24e6 credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }}