From 10d99eb873a1347b505cce581929354013549b88 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Sat, 6 Aug 2022 17:31:33 -0400 Subject: [PATCH 01/17] feat: add dockerfiles for building images from repo Signed-off-by: Daniel Bluhm --- docker/Dockerfile | 83 ++++++++++++++++++ docker/Dockerfile.indy | 43 +++++++++ docker/Dockerfile.indy-base | 170 ++++++++++++++++++++++++++++++++++++ docker/Dockerfile.test-indy | 2 +- docker/Makefile | 51 +++++++++++ 5 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.indy create mode 100644 docker/Dockerfile.indy-base create mode 100644 docker/Makefile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..3fc1899b04 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,83 @@ +ARG python_version=3.6.13 +FROM python:${python_version}-slim-buster + +ARG uid=1001 +ARG user=aries +ARG acapy_version +ARG acapy_reqs +ARG git_egg_ref + +ENV HOME="/home/$user" \ + APP_ROOT="$HOME" \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + PIP_NO_CACHE_DIR=off \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + RUST_LOG=warning \ + SHELL=/bin/bash \ + SUMMARY="aries-cloudagent image" \ + DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ + This image layers the python implementation of aries-cloudagent $acapy_version. Based on Debian Buster." + +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ + io.k8s.description="$DESCRIPTION" \ + io.k8s.display-name="aries-cloudagent $acapy_version" \ + name="aries-cloudagent" \ + version="$acapy_version" \ + maintainer="" + +# Add aries user +RUN useradd -U -ms /bin/bash -u $uid $user + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + bzip2 \ + curl \ + git \ + less \ + libffi6 \ + libgmp10 \ + liblzma5 \ + libncurses5 \ + libncursesw5 \ + libsecp256k1-0 \ + libzmq5 \ + net-tools \ + openssl \ + sqlite3 \ + vim-tiny \ + zlib1g && \ + rm -rf /var/lib/apt/lists/* /usr/share/doc/* + +WORKDIR $HOME + +# Add local binaries and aliases to path +ENV PATH="$HOME/.local/bin:$PATH" + +# - In order to drop the root user, we have to make some directories writable +# to the root group as OpenShift default security model is to run the container +# under random UID. +RUN usermod -a -G 0 $user + +# Create standard directories to allow volume mounting and set permissions +# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching +RUN mkdir -p \ + $HOME/.aries_cloudagent \ + $HOME/.cache/pip/http \ + $HOME/ledger/sandbox/data \ + $HOME/log + +# The root group needs access the directories under $HOME for the container to function in OpenShift. +# Also ensure the permissions on the python 'site-packages' folder are set correctly. +RUN chmod -R ug+rw $HOME/log $HOME/ledger $HOME/.aries_cloudagent $HOME/.cache + +RUN pip install --no-cache-dir ${git_egg_ref}aries-cloudagent${acapy_reqs}==${acapy_version} + +USER $user + +ENTRYPOINT ["aca-py"] diff --git a/docker/Dockerfile.indy b/docker/Dockerfile.indy new file mode 100644 index 0000000000..681921f8ce --- /dev/null +++ b/docker/Dockerfile.indy @@ -0,0 +1,43 @@ +ARG python_version +ARG indy_version +FROM ghcr.io/hyperledger/indy-python:py${python_version}-${indy_version} + +ARG uid=1001 +ARG user=indy +ARG acapy_version +ARG acapy_reqs +ARG git_egg_ref + +ENV HOME="/home/$user" \ + APP_ROOT="$HOME" \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + PIP_NO_CACHE_DIR=off \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + RUST_LOG=warning \ + SHELL=/bin/bash \ + SUMMARY="aries-cloudagent image" \ + DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ + This image layers the python implementation of aries-cloudagent $acapy_version. Based on indy-python, \ + this image includes indy-sdk and supporting libraries." + +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ + io.k8s.description="$DESCRIPTION" \ + io.k8s.display-name="aries-cloudagent $acapy_version" \ + name="aries-cloudagent" \ + version="$acapy_version" \ + maintainer="" + +# Create standard directories to allow volume mounting and set permissions +# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching +RUN mkdir -p $HOME/.aries_cloudagent + +# The root group needs access the directories under $HOME/.indy_client for the container to function in OpenShift. +# Also ensure the permissions on the python 'site-packages' folder are set correctly. +RUN chmod -R ug+rw $HOME/.aries_cloudagent + +RUN pip install --no-cache-dir ${git_egg_ref}aries-cloudagent${acapy_reqs}==${acapy_version} + +ENTRYPOINT ["aca-py"] diff --git a/docker/Dockerfile.indy-base b/docker/Dockerfile.indy-base new file mode 100644 index 0000000000..fb53a59b81 --- /dev/null +++ b/docker/Dockerfile.indy-base @@ -0,0 +1,170 @@ +ARG python_version=3.6.13 +ARG rust_version=1.46 +FROM rust:${rust_version}-slim-buster as builder + +ARG user=indy +ENV HOME="/home/$user" +WORKDIR $HOME +RUN mkdir -p .local/bin .local/etc .local/lib + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + automake \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + libbz2-dev \ + libffi-dev \ + libgmp-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libsecp256k1-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libzmq3-dev \ + pkg-config \ + zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +# set to --release for smaller, optimized library +ARG indy_build_flags=--release + +ARG indy_sdk_url + +# make local libs and binaries accessible +ENV PATH="$HOME/.local/bin:$PATH" +ENV LIBRARY_PATH="$HOME/.local/lib:$LIBRARY_PATH" + +# Download and extract indy-sdk +RUN mkdir indy-sdk && \ + curl "${indy_sdk_url}" | tar -xz -C indy-sdk + +# Build and install indy-sdk +WORKDIR $HOME/indy-sdk +RUN cd indy-sdk*/libindy && \ + cargo build ${indy_build_flags} && \ + cp target/*/libindy.so "$HOME/.local/lib" && \ + cargo clean + +# Package python3-indy +RUN tar czvf ../python3-indy.tgz -C indy-sdk*/wrappers/python . + +# grab the latest sdk code for the postgres plug-in +WORKDIR $HOME +ARG indy_postgres_url=${indy_sdk_url} +RUN mkdir indy-postgres && \ + curl "${indy_postgres_url}" | tar -xz -C indy-postgres + +# Build and install postgres_storage plugin +WORKDIR $HOME/indy-postgres +RUN cd indy-sdk*/experimental/plugins/postgres_storage && \ + cargo build ${indy_build_flags} && \ + cp target/*/libindystrgpostgres.so "$HOME/.local/lib" && \ + cargo clean + +# Clean up SDK +WORKDIR $HOME +RUN rm -rf indy-sdk indy-postgres + +## Start fresh (new image) ## +FROM python:${python_version}-slim-buster + + +ARG uid=1001 +ARG user=indy +ARG indy_version + +ENV HOME="/home/$user" \ + APP_ROOT="$HOME" \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + PIP_NO_CACHE_DIR=off \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + RUST_LOG=warning \ + SHELL=/bin/bash \ + SUMMARY="indy-python base image" \ + DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ + This image provides all the necessary dependencies to use the indy-sdk in python. Based on Debian Buster." + +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ + io.k8s.description="$DESCRIPTION" \ + io.k8s.display-name="indy-python $indy_version" \ + name="indy-python" \ + version="$indy_version" \ + maintainer="" + +# Add indy user +RUN useradd -U -ms /bin/bash -u $uid $user + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + bzip2 \ + curl \ + git \ + less \ + libffi6 \ + libgmp10 \ + liblzma5 \ + libncurses5 \ + libncursesw5 \ + libsecp256k1-0 \ + libzmq5 \ + net-tools \ + openssl \ + sqlite3 \ + vim-tiny \ + zlib1g && \ + rm -rf /var/lib/apt/lists/* /usr/share/doc/* + +WORKDIR $HOME + +# Copy build results +COPY --from=builder --chown=$user:$user $HOME . + +RUN mkdir -p $HOME/.local/bin + +# Add local binaries and aliases to path +ENV PATH="$HOME/.local/bin:$PATH" + +# Make libraries resolvable by python +ENV LD_LIBRARY_PATH="$HOME/.local/lib:$LD_LIBRARY_PATH" +RUN echo "$HOME/.local/lib" > /etc/ld.so.conf.d/local.conf && ldconfig + +# Install python3-indy +RUN pip install --no-cache-dir python3-indy.tgz && rm python3-indy.tgz + +# - In order to drop the root user, we have to make some directories writable +# to the root group as OpenShift default security model is to run the container +# under random UID. +RUN usermod -a -G 0 $user + +# Create standard directories to allow volume mounting and set permissions +# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching +RUN mkdir -p \ + $HOME/.cache/pip/http \ + $HOME/.indy-cli/networks \ + $HOME/.indy_client/wallet \ + $HOME/.indy_client/pool \ + $HOME/.indy_client/ledger-cache \ + $HOME/ledger/sandbox/data \ + $HOME/log + +# The root group needs access the directories under $HOME/.indy_client for the container to function in OpenShift. +# Also ensure the permissions on the python 'site-packages' folder are set correctly. +RUN chown -R $user:root $HOME/.indy_client \ + && chmod -R ug+rw $HOME/log $HOME/ledger $HOME/.cache $HOME/.indy_client + +USER $user + +CMD ["bash"] diff --git a/docker/Dockerfile.test-indy b/docker/Dockerfile.test-indy index 047b19187e..1acafb7d56 100644 --- a/docker/Dockerfile.test-indy +++ b/docker/Dockerfile.test-indy @@ -1,4 +1,4 @@ -FROM bcgovimages/von-image:py36-1.15-1 +FROM ghcr.io/hyperledger/indy-python:py3.6-1.16.0 USER indy diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 0000000000..c5ecf7c398 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,51 @@ +# A simple makefile purely to demonstrate building the new docker images + +CONTAINER_RUNTIME ?= docker +IMAGE_NAME=ghcr.io/hyperledger/aries-cloudagent-python +PYTHON_VERSION=3.6.13 +PYTHON_VERSION_MAJ_MIN=3.6 +RUST_VERSION=1.46 +ACAPY_VERSION=0.7.4 +ACAPY_REQS=[askar,bbs] +INDY_VERSION=1.16.0 +INDY_SDK_URL=https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/v$(INDY_VERSION) +INDY_IMAGE_NAME=ghcr.io/hyperledger/indy-python + +all: indy-python indy standard + +indy-python: + $(CONTAINER_RUNTIME) build -t $(INDY_IMAGE_NAME):latest \ + --build-arg python_version=$(PYTHON_VERSION) \ + --build-arg rust_version=$(RUST_VERSION) \ + --build-arg indy_version=$(INDY_VERSION) \ + --build-arg indy_sdk_url=$(INDY_SDK_URL) \ + -f Dockerfile.indy-base . + $(CONTAINER_RUNTIME) tag $(INDY_IMAGE_NAME):latest \ + $(INDY_IMAGE_NAME):py$(PYTHON_VERSION)-$(INDY_VERSION) + $(CONTAINER_RUNTIME) tag $(INDY_IMAGE_NAME):latest \ + $(INDY_IMAGE_NAME):py$(PYTHON_VERSION_MAJ_MIN)-$(INDY_VERSION) + +indy: + $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):indy-latest \ + --build-arg python_version=$(PYTHON_VERSION) \ + --build-arg indy_version=$(INDY_VERSION) \ + --build-arg acapy_version=$(ACAPY_VERSION) \ + --build-arg acapy_reqs=$(ACAPY_REQS) \ + -f Dockerfile.indy . + $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):indy-latest \ + $(IMAGE_NAME):py$(PYTHON_VERSION)-indy-$(INDY_VERSION)-$(ACAPY_VERSION) + $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):indy-latest \ + $(IMAGE_NAME):py$(PYTHON_VERSION_MAJ_MIN)-indy-$(INDY_VERSION)-$(ACAPY_VERSION) + +standard: + $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):latest \ + --build-arg python_version=$(PYTHON_VERSION) \ + --build-arg acapy_version=$(ACAPY_VERSION) \ + --build-arg acapy_reqs=$(ACAPY_REQS) \ + -f Dockerfile . + $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):latest \ + $(IMAGE_NAME):py$(PYTHON_VERSION)-$(ACAPY_VERSION) + $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):latest \ + $(IMAGE_NAME):py$(PYTHON_VERSION_MAJ_MIN)-$(ACAPY_VERSION) + +.PHONY: all indy-python indy standard From 978e35bc705a02b43d621d27a93f35357243e834 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Tue, 16 Aug 2022 12:57:00 -0400 Subject: [PATCH 02/17] test: add gha for testing with in repo dockerfiles Signed-off-by: Daniel Bluhm --- .dockerignore | 4 +- .github/workflows/nightly.yml | 84 +++++++++++++ .github/workflows/tests-indy.yml | 198 +++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 38 ++++++ docker/Dockerfile | 16 ++- docker/Dockerfile.test | 9 +- docker/Dockerfile.test-indy | 7 +- 7 files changed, 346 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/tests-indy.yml create mode 100644 .github/workflows/tests.yml diff --git a/.dockerignore b/.dockerignore index 910edaa6ff..7ea06888de 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,6 @@ build docs dist test-reports -.python-version \ No newline at end of file +.python-version +docker +env diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..4a02ae3f1e --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,84 @@ +name: Nightly Build +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +env: + NAME: aries-cloudagent-python + PYTHON_VERSION: 3.7 + +jobs: + nightly: + name: Nightly + runs-on: ubuntu-latest + steps: + - name: Gather image info + id: info + run: | + echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" + + - name: Check image exists + id: image-exists + uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef + with: + tag: ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }}:py${{ env.PYTHON_VERSION }}-nightly-${{ github.sha }} + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + if: steps.image-exists.outputs.exists != 'true' + + - name: Cache Docker layers + if: steps.image-exists.outputs.exists != 'true' + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Set up Docker Buildx + if: steps.image-exists.outputs.exists != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + if: steps.image-exists.outputs.exists != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Base Image Metadata + if: steps.image-exists.outputs.exists != 'true' + id: base-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }} + tags: | + type=raw,value=py${{ env.PYTHON_VERSION }}-nightly + type=sha,format=long,prefix=py${{ env.PYTHON_VERSION }}-nightly- + + - name: Build and Push Base Image to ghcr.io + if: steps.image-exists.outputs.exists != 'true' + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile + tags: ${{ steps.base-meta.outputs.tags }} + labels: ${{ steps.base-meta.outputs.labels }} + build-args: | + python_version=${{ env.PYTHON_VERSION }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + if: steps.image-exists.outputs.exists != 'true' + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache-base diff --git a/.github/workflows/tests-indy.yml b/.github/workflows/tests-indy.yml new file mode 100644 index 0000000000..86d5c38601 --- /dev/null +++ b/.github/workflows/tests-indy.yml @@ -0,0 +1,198 @@ +name: Tests (Indy) +on: + pull_request: + +env: + INDY_VERSION: 1.16.0 + +jobs: + info: + name: Gather image info + runs-on: ubuntu-latest + outputs: + repo-owner: ${{ steps.info.outputs.owner-lc }} + indy-version: ${{ steps.info.outputs.indy-version }} + indy-sdk-url: ${{ steps.info.outputs.indy-sdk-url }} + base-dep-hash: ${{ steps.info.outputs.base-hash }} + test-dep-hash: ${{ steps.info.outputs.test-hash }} + steps: + - uses: actions/checkout@v3 + - name: Gather image info + id: info + run: | + echo "::set-output name=owner-lc::${GITHUB_REPOSITORY_OWNER,,}" + echo "::set-output name=indy-version::${{env.INDY_VERSION}}" + echo "::set-output name=indy-sdk-url::https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/v${{ env.INDY_VERSION }}" + echo "::set-output name=base-hash::${{ hashFiles('docker/Dockerfile.indy-base') }}" + echo "::set-output name=test-hash::${{ hashFiles('requirements*.txt', 'docker/Dockerfile.test-indy') }}" + + base-image: + name: Publish base image + needs: info + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - name: Check image exists + id: image-exists + uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef + with: + tag: ghcr.io/${{ needs.info.outputs.repo-owner }}/indy-python-test:py${{ matrix.python-version }}-${{ env.INDY_VERSION }}-${{ needs.info.outputs.base-dep-hash }} + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + if: steps.image-exists.outputs.exists != 'true' + + - name: Cache Docker layers + if: steps.image-exists.outputs.exists != 'true' + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-base + key: ${{ runner.os }}-buildx-base-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-base- + + - name: Set up Docker Buildx + if: steps.image-exists.outputs.exists != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + if: steps.image-exists.outputs.exists != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Base Image Metadata + if: steps.image-exists.outputs.exists != 'true' + id: base-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ needs.info.outputs.repo-owner }}/indy-python-test + tags: | + type=raw,value=py${{ matrix.python-version }}-${{ env.INDY_VERSION }}-${{ needs.info.outputs.base-dep-hash }} + + - name: Build and Push Base Image to ghcr.io + if: steps.image-exists.outputs.exists != 'true' + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile.indy-base + tags: ${{ steps.base-meta.outputs.tags }} + labels: ${{ steps.base-meta.outputs.labels }} + build-args: | + python_version=${{ matrix.python-version }} + indy_version=${{ needs.info.outputs.indy_version }} + indy_sdk_url=${{ needs.info.outputs.indy-sdk-url }} + cache-from: type=local,src=/tmp/.buildx-cache-base + cache-to: type=local,dest=/tmp/.buildx-cache-base-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + if: steps.image-exists.outputs.exists != 'true' + run: | + rm -rf /tmp/.buildx-cache-base + mv /tmp/.buildx-cache-base-new /tmp/.buildx-cache-base + + + test-image: + name: Publish test image + needs: ["info", "base-image"] + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + + - name: Check image exists + id: image-exists + uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef + with: + tag: ghcr.io/${{ needs.info.outputs.repo-owner }}/acapy-test:py${{ matrix.python-version }}-${{ needs.info.outputs.indy-version }}-${{ needs.info.outputs.test-dep-hash }} + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + if: steps.image-exists.outputs.exists != 'true' + + - name: Cache Docker layers + if: steps.image-exists.outputs.exists != 'true' + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-test + key: ${{ runner.os }}-buildx-test-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-test- + + - name: Set up Docker Buildx + if: steps.image-exists.outputs.exists != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + if: steps.image-exists.outputs.exists != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Test Image Metadata + if: steps.image-exists.outputs.exists != 'true' + id: test-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ needs.info.outputs.repo-owner }}/acapy-test + tags: | + type=raw,value=py${{ matrix.python-version }}-${{ env.INDY_VERSION }}-${{ needs.info.outputs.test-dep-hash }} + + - name: Build and Push Test Image to ghcr.io + if: steps.image-exists.outputs.exists != 'true' + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile.test-indy + tags: ${{ steps.test-meta.outputs.tags }} + labels: ${{ steps.test-meta.outputs.labels }} + build-args: | + base_image=ghcr.io/${{ needs.info.outputs.repo-owner }}/indy-python-test:py${{ matrix.python-version }}-${{ needs.info.outputs.indy-version }}-${{ needs.info.outputs.base-dep-hash }} + cache-from: type=local,src=/tmp/.buildx-cache-test + cache-to: type=local,dest=/tmp/.buildx-cache-test-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + if: steps.image-exists.outputs.exists != 'true' + run: | + rm -rf /tmp/.buildx-cache-test + mv /tmp/.buildx-cache-test-new /tmp/.buildx-cache-test + + tests: + name: Tests (Indy) + needs: + - info + - base-image + - test-image + runs-on: ubuntu-latest + container: ghcr.io/${{ needs.info.outputs.repo-owner }}/acapy-test:py${{ matrix.python-version }}-${{ needs.info.outputs.indy-version }}-${{ needs.info.outputs.test-dep-hash }} + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Run pytest + run: | + pytest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..6205a865fc --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,38 @@ +name: Tests + +on: + pull_request: + +jobs: + tests: + name: Tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: 'requirements*.txt' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip3 install --no-cache-dir \ + -r requirements.txt \ + -r requirements.askar.txt \ + -r requirements.bbs.txt \ + -r requirements.dev.txt + - name: Tests + run: | + pytest --junitxml=pytest.xml + - name: Test Report + uses: mikepenz/action-junit-report@v3 + if: always() + with: + report_paths: pytest.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index 3fc1899b04..0a40fbbe40 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,11 +1,19 @@ ARG python_version=3.6.13 -FROM python:${python_version}-slim-buster +FROM python:${python_version}-slim-buster AS build + +WORKDIR /src + +ADD . . + +RUN pip install setuptools wheel +RUN python setup.py sdist bdist_wheel + +FROM python:${python_version}-slim-buster AS main ARG uid=1001 ARG user=aries ARG acapy_version ARG acapy_reqs -ARG git_egg_ref ENV HOME="/home/$user" \ APP_ROOT="$HOME" \ @@ -76,7 +84,9 @@ RUN mkdir -p \ # Also ensure the permissions on the python 'site-packages' folder are set correctly. RUN chmod -R ug+rw $HOME/log $HOME/ledger $HOME/.aries_cloudagent $HOME/.cache -RUN pip install --no-cache-dir ${git_egg_ref}aries-cloudagent${acapy_reqs}==${acapy_version} +COPY --from=build /src/dist/aries_cloudagent*.whl . + +RUN pip install --no-cache-dir --find-links=. aries_cloudagent${acapy_reqs} && rm aries_cloudagent*.whl USER $user diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index e949e9274f..6a1b4df76a 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,11 +1,10 @@ -FROM python:3.6.13 +ARG python_version=3.6.13 +FROM python:${python_version}-slim-buster RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ - python3 \ - python3-pip \ - python3-setuptools \ libsodium23 && \ + apt-get clean && \ rm -rf /var/lib/apt/lists/* WORKDIR /usr/src/app @@ -20,4 +19,4 @@ RUN pip3 install --no-cache-dir \ ADD . . -ENTRYPOINT ["/bin/bash", "-c", "pytest \"$@\"", "--"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "-c", "pytest \"$@\"", "--"] diff --git a/docker/Dockerfile.test-indy b/docker/Dockerfile.test-indy index 1acafb7d56..6b38d885e6 100644 --- a/docker/Dockerfile.test-indy +++ b/docker/Dockerfile.test-indy @@ -1,4 +1,7 @@ -FROM ghcr.io/hyperledger/indy-python:py3.6-1.16.0 +ARG python_version=3.6.13 +ARG indy_version=1.16.0 +ARG base_image=ghcr.io/hyperledger/indy-python:py${python_version}-${indy_version} +FROM ${base_image} USER indy @@ -10,6 +13,7 @@ RUN mkdir -p test-reports && chown -R indy:indy test-reports && chmod -R ug+rw t ADD requirements*.txt ./ +USER root RUN pip3 install --no-cache-dir \ -r requirements.txt \ -r requirements.askar.txt \ @@ -17,5 +21,6 @@ RUN pip3 install --no-cache-dir \ -r requirements.dev.txt ADD --chown=indy:root . . +USER indy ENTRYPOINT ["/bin/bash", "-c", "pytest \"$@\"", "--"] From 6c641fedc57f0901fa4facdc8f74f1de5c4217df Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Mon, 15 Aug 2022 17:53:18 -0400 Subject: [PATCH 03/17] feat: add nightly workflow for indy aca-py Signed-off-by: Daniel Bluhm --- .github/workflows/nightly-indy.yml | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/nightly-indy.yml diff --git a/.github/workflows/nightly-indy.yml b/.github/workflows/nightly-indy.yml new file mode 100644 index 0000000000..82288898b0 --- /dev/null +++ b/.github/workflows/nightly-indy.yml @@ -0,0 +1,86 @@ +name: Nightly Build (Indy) +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +env: + NAME: aries-cloudagent-python + PYTHON_VERSION: 3.7 + INDY_VERSION: 1.16.0 + +jobs: + nightly: + name: Nightly (Indy) + runs-on: ubuntu-latest + steps: + - name: Gather image info + id: info + run: | + echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" + + - name: Check image exists + id: image-exists + uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef + with: + tag: ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }}:py${{ env.PYTHON_VERSION }}-indy-${{ env.INDY_VERSION }}-nightly-${{ github.sha }} + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + if: steps.image-exists.outputs.exists != 'true' + + - name: Cache Docker layers + if: steps.image-exists.outputs.exists != 'true' + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Set up Docker Buildx + if: steps.image-exists.outputs.exists != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + if: steps.image-exists.outputs.exists != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Base Image Metadata + if: steps.image-exists.outputs.exists != 'true' + id: base-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }} + tags: | + type=raw,value=py${{ env.PYTHON_VERSION }}-indy-${{ env.INDY_VERSION }}-nightly + type=sha,format=long,prefix=py${{ env.PYTHON_VERSION }}-indy-${{ env.INDY_VERSION }}-nightly- + + - name: Build and Push Base Image to ghcr.io + if: steps.image-exists.outputs.exists != 'true' + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile.indy + tags: ${{ steps.base-meta.outputs.tags }} + labels: ${{ steps.base-meta.outputs.labels }} + build-args: | + python_version=${{ env.PYTHON_VERSION }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + if: steps.image-exists.outputs.exists != 'true' + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache-base + From 2fe605add368a5e80aad238195d73d53daec9855 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Mon, 15 Aug 2022 17:53:43 -0400 Subject: [PATCH 04/17] feat: add publish indy-python workflow Signed-off-by: Daniel Bluhm --- .github/workflows/publish-indy-python.yml | 101 ++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 .github/workflows/publish-indy-python.yml diff --git a/.github/workflows/publish-indy-python.yml b/.github/workflows/publish-indy-python.yml new file mode 100644 index 0000000000..520eacbf9f --- /dev/null +++ b/.github/workflows/publish-indy-python.yml @@ -0,0 +1,101 @@ +name: Publish Indy Python +on: + workflow_dispatch: + inputs: + indy_sdk_url: + description: 'Indy SDK download URL' + required: false + type: string + indy_postgres_url: + description: 'Indy postgres download URL' + required: false + type: string + indy_version: + description: 'Indy SDK Version' + required: false + type: string + default: '1.16.0' + tag: + description: 'Image tag' + required: false + type: string + +env: + INDY_SDK_TAG_URL: "https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/" + +jobs: + publish-image: + strategy: + fail-fast: false + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10'] + + name: Publish Indy Python + runs-on: ubuntu-latest + steps: + - name: Gather image info + id: info + run: | + echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" + + [ -n "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ inputs.indy_sdk_url }}" + [ -z "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ env.INDY_SDK_TAG_URL }}v${{ env.INDY_VERSION }}" + + [ -n "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ inputs.indy_postgres_url }}" + [ -z "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ env.INDY_SDK_TAG_URL }}v${{ env.INDY_VERSION }}" + + [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" + [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-${{ inputs.indy_version }}" + + - uses: actions/checkout@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-base + key: ${{ runner.os }}-buildx-base-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-base- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Base Image Metadata + id: base-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/indy-python + tags: | + type=raw,value=${{ steps.info.outputs.tag }} + + - name: Build and Push Base Image to ghcr.io + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile.indy-base + tags: ${{ steps.base-meta.outputs.tags }} + labels: ${{ steps.base-meta.outputs.labels }} + build-args: | + python_version=${{ matrix.python-version }} + indy_version=${{ inputs.indy_version }} + indy_sdk_url=${{ steps.info.outputs.indy-sdk-url }} + indy_postgres_url=${{ steps.info.outputs.indy-postgres-url }} + cache-from: type=local,src=/tmp/.buildx-cache-base + cache-to: type=local,dest=/tmp/.buildx-cache-base-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-base + mv /tmp/.buildx-cache-base-new /tmp/.buildx-cache-base From 9603484d02c425b16f663a111211839740538708 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Mon, 15 Aug 2022 17:54:03 -0400 Subject: [PATCH 05/17] feat: add publish workflow Signed-off-by: Daniel Bluhm --- .github/workflows/publish.yml | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..8f1e2197ee --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,86 @@ +name: Publish Aries Cloud Agent - Python +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag' + required: false + type: string + +env: + ACAPY_REQS: '[askar,bbs]' + + +jobs: + publish-image: + strategy: + fail-fast: false + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10'] + + name: Publish Aries Cloud Agent - Python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Gather image info + id: info + run: | + echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" + echo "::set-output name=acapy-version::$(sed -ne 's/__version__ = \"\([0-9.]\+\)\"/\1/p' aries_cloudagent/version.py)" + + - name: Tag image + id: tag + run: | + [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" + [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-${{ steps.info.outputs.acapy-version }}" + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-base-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-base- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Base Image Metadata + id: base-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python + tags: | + type=raw,value=${{ steps.tag.outputs.tag }} + + - name: Build and Push Base Image to ghcr.io + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile + tags: ${{ steps.base-meta.outputs.tags }} + labels: ${{ steps.base-meta.outputs.labels }} + build-args: | + python_version=${{ matrix.python-version }} + acapy_version=${{ steps.info.outputs.acapy-version }} + acapy_reqs=${{ env.ACAPY_REQS }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 4c9795fe4c67c3250d453cc54170757a45baf664 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Tue, 16 Aug 2022 10:37:13 -0400 Subject: [PATCH 06/17] feat: add actions for publishing images Signed-off-by: Daniel Bluhm fix: url construction for indy sdk download Signed-off-by: Daniel Bluhm fix: bad labels on steps Signed-off-by: Daniel Bluhm --- .github/workflows/publish-indy-python.yml | 8 +- .github/workflows/publish-indy.yml | 93 +++++++++++++++++++++++ .github/workflows/publish.yml | 8 +- docker/Dockerfile.indy | 19 ++++- 4 files changed, 116 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/publish-indy.yml diff --git a/.github/workflows/publish-indy-python.yml b/.github/workflows/publish-indy-python.yml index 520eacbf9f..f2b0bea15b 100644 --- a/.github/workflows/publish-indy-python.yml +++ b/.github/workflows/publish-indy-python.yml @@ -39,10 +39,10 @@ jobs: echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" [ -n "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ inputs.indy_sdk_url }}" - [ -z "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ env.INDY_SDK_TAG_URL }}v${{ env.INDY_VERSION }}" + [ -z "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ env.INDY_SDK_TAG_URL }}v${{ inputs.indy_version }}" [ -n "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ inputs.indy_postgres_url }}" - [ -z "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ env.INDY_SDK_TAG_URL }}v${{ env.INDY_VERSION }}" + [ -z "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ env.INDY_SDK_TAG_URL }}v${{ inputs.indy_version }}" [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-${{ inputs.indy_version }}" @@ -67,7 +67,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Base Image Metadata + - name: Setup Image Metadata id: base-meta uses: docker/metadata-action@v3 with: @@ -76,7 +76,7 @@ jobs: tags: | type=raw,value=${{ steps.info.outputs.tag }} - - name: Build and Push Base Image to ghcr.io + - name: Build and Push Image to ghcr.io uses: docker/build-push-action@v3 with: push: true diff --git a/.github/workflows/publish-indy.yml b/.github/workflows/publish-indy.yml new file mode 100644 index 0000000000..058187d030 --- /dev/null +++ b/.github/workflows/publish-indy.yml @@ -0,0 +1,93 @@ +name: Publish ACA-Py (Indy) +on: + workflow_dispatch: + inputs: + indy_version: + description: 'Indy SDK Version' + required: false + type: string + default: '1.16.0' + tag: + description: 'Image tag' + required: false + type: string + +env: + ACAPY_REQS: '[askar,bbs]' + + +jobs: + publish-image: + strategy: + fail-fast: false + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10'] + + name: Publish (Indy) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Gather image info + id: info + run: | + echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" + echo "::set-output name=acapy-version::$(sed -ne 's/__version__ = \"\([0-9.]\+\)\"/\1/p' aries_cloudagent/version.py)" + + - name: Tag image + id: tag + run: | + [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" + [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-indy-${{ inputs.indy_version }}-${{ steps.info.outputs.acapy-version }}" + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-base-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-base- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Image Metadata + id: base-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python + tags: | + type=raw,value=${{ steps.tag.outputs.tag }} + + - name: Build and Push Image to ghcr.io + uses: docker/build-push-action@v3 + with: + push: true + context: . + file: docker/Dockerfile.indy + tags: ${{ steps.base-meta.outputs.tags }} + labels: ${{ steps.base-meta.outputs.labels }} + build-args: | + python_version=${{ matrix.python-version }} + indy_version=${{ inputs.indy_version }} + acapy_version=${{ steps.info.outputs.acapy-version }} + acapy_reqs=${{ env.ACAPY_REQS }} + org=${{ steps.info.outputs.repo-owner }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8f1e2197ee..15449f4350 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Publish Aries Cloud Agent - Python +name: Publish ACA-Py on: workflow_dispatch: inputs: @@ -18,7 +18,7 @@ jobs: matrix: python-version: ['3.7', '3.8', '3.9', '3.10'] - name: Publish Aries Cloud Agent - Python + name: Publish runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -53,7 +53,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Base Image Metadata + - name: Setup Image Metadata id: base-meta uses: docker/metadata-action@v3 with: @@ -62,7 +62,7 @@ jobs: tags: | type=raw,value=${{ steps.tag.outputs.tag }} - - name: Build and Push Base Image to ghcr.io + - name: Build and Push Image to ghcr.io uses: docker/build-push-action@v3 with: push: true diff --git a/docker/Dockerfile.indy b/docker/Dockerfile.indy index 681921f8ce..9fd35fd5b6 100644 --- a/docker/Dockerfile.indy +++ b/docker/Dockerfile.indy @@ -1,12 +1,21 @@ -ARG python_version +ARG python_version=3.6.13 ARG indy_version -FROM ghcr.io/hyperledger/indy-python:py${python_version}-${indy_version} +ARG org=hyperledger +FROM python:${python_version}-slim-buster AS build + +WORKDIR /src + +ADD . . + +RUN pip install setuptools wheel +RUN python setup.py sdist bdist_wheel + +FROM ghcr.io/${org}/indy-python:py${python_version}-${indy_version} AS main ARG uid=1001 ARG user=indy ARG acapy_version ARG acapy_reqs -ARG git_egg_ref ENV HOME="/home/$user" \ APP_ROOT="$HOME" \ @@ -38,6 +47,8 @@ RUN mkdir -p $HOME/.aries_cloudagent # Also ensure the permissions on the python 'site-packages' folder are set correctly. RUN chmod -R ug+rw $HOME/.aries_cloudagent -RUN pip install --no-cache-dir ${git_egg_ref}aries-cloudagent${acapy_reqs}==${acapy_version} +COPY --from=build /src/dist/aries_cloudagent*.whl . + +RUN pip install --no-cache-dir --find-links=. aries_cloudagent${acapy_reqs} && rm aries_cloudagent*.whl ENTRYPOINT ["aca-py"] From 147262998c62a8366fc8527ed2da5c7757351ca3 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Sat, 17 Sep 2022 12:32:08 -0400 Subject: [PATCH 07/17] docs: description of images Signed-off-by: Daniel Bluhm --- ContainerImagesAndGithubActions.md | 118 +++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 ContainerImagesAndGithubActions.md diff --git a/ContainerImagesAndGithubActions.md b/ContainerImagesAndGithubActions.md new file mode 100644 index 0000000000..27d67d0363 --- /dev/null +++ b/ContainerImagesAndGithubActions.md @@ -0,0 +1,118 @@ +# Container Images and Github Actions + +Aries Cloud Agent - Python is most frequently deployed using containers. From +the first release of ACA-Py up through 0.7.4, much of the community has built +their Aries stack using the container images graciously provided by BC Gov and +hosted through their `bcgovimages` docker hub account. These images have been +critical to the adoption of not only ACA-Py but also Hyperledger Aries and SSI +more generally. + +Recognizing how critical these images are to the success of ACA-Py and +consistent with Hyperledger's commitment to open collaboration, container images +are now built and published directly from the Aries Cloud Agent - Python project +repository and made available through the [Github Packages Container +Registry](https://ghcr.io). + + +## Images + +The following images are built from this project + +- `ghcr.io/hyperledger/aries-cloudagent-python` - multiple variants are built + from this project; see [Tags](#tags). +- `ghcr.io/hyperledger/indy-python` - this image is used as a base for the + ACA-Py Indy variant (see [Tags](#tags)). This may be moved to a more + appropriate project in the future. + + +### Tags + +ACA-Py is a foundation for building decentralized identity applications; to this +end, there are multiple variants of ACA-Py built to suit the needs of a variety +of environments and workflows. There are currently two main variants: + +- "Standard" - The default configuration of ACA-Py, including: + - Aries Askar for secure storage + - Indy VDR for Indy ledger communication + - Indy Shared Libraries for AnonCreds +- "Indy" - The legacy configuration of ACA-Py, including: + - Indy SDK Wallet for secure storage + - Indy SDK Ledger for Indy ledger communication + - Indy SDK for AnonCreds + +These two image variants are largely distinguished by providers for Indy Network +and AnonCreds support. The Standard variant is recommended for new projects. +Migration from an Indy based image (whether the new Indy image variant or the +original BC Gov images) to the Standard image is outside of the scope of this +document. + +The ACA-Py images built by this project are tagged to indicate which of the +above variants it is. Other tags are also generated for use by developers. + +Below is a table of all generated images and their tags: + +Tag | Variant | Example | Description | +------------------------|----------|--------------------------|-------------------------------------------------------------------------------------------------| +py3.7-X.Y.Z | Standard | py3.7-0.7.4 | Standard image variant built on Python 3.7 for ACA-Py version X.Y.Z | +py3.8-X.Y.Z | Standard | py3.8-0.7.4 | Standard image variant built on Python 3.8 for ACA-Py version X.Y.Z | +py3.9-X.Y.Z | Standard | py3.9-0.7.4 | Standard image variant built on Python 3.9 for ACA-Py version X.Y.Z | +py3.10-X.Y.Z | Standard | py3.10-0.7.4 | Standard image variant built on Python 3.10 for ACA-Py version X.Y.Z | +py3.7-indy-A.B.C-X.Y.Z | Indy | py3.7-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.7 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | +py3.8-indy-A.B.C-X.Y.Z | Indy | py3.8-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.8 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | +py3.9-indy-A.B.C-X.Y.Z | Indy | py3.9-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.9 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | +py3.10-indy-A.B.C-X.Y.Z | Indy | py3.10-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.10 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | + + +#### Indy Python + +**Image Name:** `ghcr.io/hyperledger/indy-python` + +The Indy Python image is used as a base for the Indy variant of ACA-Py. It is a +debian based image with `libindy` and the Indy SDK Python wrapper installed. + +Below is a table of all generated Indy Python images and their tags: + +Tag | Example | Description | +------------------------|--------------------------|-----------------------------------------| +py3.7-X.Y.Z | py3.7-1.16.0 | Python 3.7 with Indy SDK version X.Y.Z | +py3.8-X.Y.Z | py3.8-1.16.0 | Python 3.8 with Indy SDK version X.Y.Z | +py3.9-X.Y.Z | py3.9-1.16.0 | Python 3.9 with Indy SDK version X.Y.Z | +py3.10-X.Y.Z | py3.10-1.16.0 | Python 3.10 with Indy SDK version X.Y.Z | + + +#### Nightly + +The Github Actions will also produce Nightly builds of ACA-Py. If a nightly +build at the current hash of the repo doesn't yet exist, GHA will build a +standard and Indy ACA-Py image at midnight each day. Nightly builds are produced +only for the current "active" python version. + +Below is a table of all generated Nightly images and their tags: + +Tag | Variant | Example | Description | +-------------------------------------------|----------|--------------------------------------------------------------------|---------------------------------------| +py3.7-nightly | Standard | py3.7-nightly | Standard image latest nightly | +py3.7-indy-A.B.C-nightly | Indy | py3.7-indy-1.16.0-nightly | Indy image latest nightly | +py3.7-nightly-{{ commit hash }} | Standard | py3.7-nightly-96bc6a8938f0c0e2a487a069d63bcb6c8172b320 | Standard image nightly at commit hash | +py3.7-indy-A.B.C-nightly-{{ commit hash }} | Indy | py3.7-indy-1.16.0-nightly-96bc6a8938f0c0e2a487a069d63bcb6c8172b320 | Indy image nightly at commit hash | + + +#### Testing + +The Github Actions will produce images used in CI/CD checks for Indy (Indy image +tests require `libindy` which is not available on Github runners; these tests +must be run inside of a container with `libindy`). These images are only +intended for use by these checks. + +Below is a table of all generated test images and their tags: + +Image + Tag | Description | +--------------------------------------------------------------------------------------------------------|------------------------------------| +indy-python-test:py{{python-version}}-{{indy-version}}-{{hash of indy base Dockerfile}} | Base Indy Python image for testing | +acapy-test:py{{python-version}}-{{indy-version}}-{{hash of requirements*.txt and indy test Dockerfile}} | ACA-Py test image | + +## Github Actions + +Several Github Actions are used to produce the above described images. + +**TODO:** Add descriptions of actions From 412fb179809986f6ee4a687bb6c91a255e190dba Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Sat, 17 Sep 2022 12:53:13 -0400 Subject: [PATCH 08/17] docs: add key differences section Signed-off-by: Daniel Bluhm --- ContainerImagesAndGithubActions.md | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ContainerImagesAndGithubActions.md b/ContainerImagesAndGithubActions.md index 27d67d0363..e5e8ccf277 100644 --- a/ContainerImagesAndGithubActions.md +++ b/ContainerImagesAndGithubActions.md @@ -116,3 +116,38 @@ acapy-test:py{{python-version}}-{{indy-version}}-{{hash of requirements*.txt and Several Github Actions are used to produce the above described images. **TODO:** Add descriptions of actions + +## Key Differences + +There are several key differences that should be noted between the two image +variants and between the BC Gov ACA-Py images and VON images and the images +produced by this project. + +- Standard Image + - Based on slim variant of Debian + - Does **NOT** include `libindy` + - Default user is `aries` + - Uses container's system python environment rather than `pyenv` + - Askar and Indy Shared libraries are installed through pip from + pre-compiled binaries included in the python wrappers. + - Built from repo contents +- Indy Image + - Based on slim variant of Debian + - Based on `indy-python` + - Includes `libindy` but does **NOT** include the Indy CLI + - Default user is `indy` + - Based on `indy-python` + - Uses container's system python environment rather than `pyenv` + - Askar and Indy Shared libraries are installed through pip from + pre-compiled binaries included in the python wrappers + - Built from repo contents + - Includes Indy postgres storage plugin +- `bcgovimages/aries-cloudagent` + - (Usually) based on Ubuntu + - Based on `von-image` + - Default user is `indy` + - Includes `libindy` and Indy CLI + - Uses `pyenv` + - Askar and Indy Shared libraries built from source + - Built from ACA-Py python package uploaded to PyPI + - Includes Indy postgres storage plugin From 1e85957d509be753a5383bffb60dc2ba6c4b7788 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Tue, 18 Oct 2022 22:52:09 -0400 Subject: [PATCH 09/17] Simplify actions and images * test: build and run without push Signed-off-by: Daniel Bluhm * test: further slim tests-indy Signed-off-by: Daniel Bluhm * fix: store image in docker Signed-off-by: Daniel Bluhm * fix: load true for build of images Signed-off-by: Daniel Bluhm * fix: bad workflow for tests-indy Signed-off-by: Daniel Bluhm * refactor: indy images all from one Dockerfile And don't publish indy-python Signed-off-by: Daniel Bluhm * fix: no tty in gha Signed-off-by: Daniel Bluhm * fix: clean up workflows Signed-off-by: Daniel Bluhm * fix: id of jobs and conditions Signed-off-by: Daniel Bluhm * fix: publish indy meta and condition Signed-off-by: Daniel Bluhm * fix: tags for release on indy image publish Signed-off-by: Daniel Bluhm * refactor: move local actions to .github folder Signed-off-by: Daniel Bluhm * feat: create nightly testing workflows Signed-off-by: Daniel Bluhm * ci: use reusable workflows to DRY things up Signed-off-by: Daniel Bluhm * fix: env values Signed-off-by: Daniel Bluhm * fix: env values again Signed-off-by: Daniel Bluhm * fix: don't use env Signed-off-by: Daniel Bluhm * refactor: unified pr and nightly workflows Signed-off-by: Daniel Bluhm * fix: run on python 3.6 Signed-off-by: Daniel Bluhm * docs: update docs Signed-off-by: Daniel Bluhm * chore: update label on workflow Signed-off-by: Daniel Bluhm * chore: fix typo in docs Signed-off-by: Daniel Bluhm Signed-off-by: Daniel Bluhm --- .../actions}/run-indy-tails-server/action.yml | 0 .../actions}/run-integration-tests/action.yml | 0 .../actions}/run-von-network/action.yml | 0 .github/workflows/integrationtests.yml | 2 +- .github/workflows/nightly-indy.yml | 86 ------- .github/workflows/nightly-tests.yml | 29 +++ .github/workflows/nightly.yml | 84 ------- .github/workflows/pr-tests.yml | 18 ++ .github/workflows/publish-indy-python.yml | 101 -------- .github/workflows/publish-indy.yml | 50 ++-- .github/workflows/publish.yml | 47 ++-- .github/workflows/tests-indy.yml | 183 ++------------- .github/workflows/tests.yml | 17 +- ContainerImagesAndGithubActions.md | 105 +++------ docker/Dockerfile | 2 +- docker/Dockerfile.indy | 220 +++++++++++++++++- docker/Dockerfile.indy-base | 170 -------------- scripts/run_tests_indy | 7 +- 18 files changed, 387 insertions(+), 734 deletions(-) rename {actions => .github/actions}/run-indy-tails-server/action.yml (100%) rename {actions => .github/actions}/run-integration-tests/action.yml (100%) rename {actions => .github/actions}/run-von-network/action.yml (100%) delete mode 100644 .github/workflows/nightly-indy.yml create mode 100644 .github/workflows/nightly-tests.yml delete mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/pr-tests.yml delete mode 100644 .github/workflows/publish-indy-python.yml delete mode 100644 docker/Dockerfile.indy-base diff --git a/actions/run-indy-tails-server/action.yml b/.github/actions/run-indy-tails-server/action.yml similarity index 100% rename from actions/run-indy-tails-server/action.yml rename to .github/actions/run-indy-tails-server/action.yml diff --git a/actions/run-integration-tests/action.yml b/.github/actions/run-integration-tests/action.yml similarity index 100% rename from actions/run-integration-tests/action.yml rename to .github/actions/run-integration-tests/action.yml diff --git a/actions/run-von-network/action.yml b/.github/actions/run-von-network/action.yml similarity index 100% rename from actions/run-von-network/action.yml rename to .github/actions/run-von-network/action.yml diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml index db62b14a34..404f7c562a 100644 --- a/.github/workflows/integrationtests.yml +++ b/.github/workflows/integrationtests.yml @@ -21,7 +21,7 @@ jobs: #- name: run-indy-tails-server # uses: ./acapy/actions/run-indy-tails-server - name: run-integration-tests - uses: ./acapy/actions/run-integration-tests + uses: ./acapy/.github/actions/run-integration-tests # to run with a specific set of tests include the following parameter: # with: # TEST_SCOPE: "-t @T001-RFC0037" diff --git a/.github/workflows/nightly-indy.yml b/.github/workflows/nightly-indy.yml deleted file mode 100644 index 82288898b0..0000000000 --- a/.github/workflows/nightly-indy.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Nightly Build (Indy) -on: - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -env: - NAME: aries-cloudagent-python - PYTHON_VERSION: 3.7 - INDY_VERSION: 1.16.0 - -jobs: - nightly: - name: Nightly (Indy) - runs-on: ubuntu-latest - steps: - - name: Gather image info - id: info - run: | - echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" - - - name: Check image exists - id: image-exists - uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef - with: - tag: ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }}:py${{ env.PYTHON_VERSION }}-indy-${{ env.INDY_VERSION }}-nightly-${{ github.sha }} - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/checkout@v3 - if: steps.image-exists.outputs.exists != 'true' - - - name: Cache Docker layers - if: steps.image-exists.outputs.exists != 'true' - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Set up Docker Buildx - if: steps.image-exists.outputs.exists != 'true' - uses: docker/setup-buildx-action@v1 - - - name: Log in to the GitHub Container Registry - if: steps.image-exists.outputs.exists != 'true' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Base Image Metadata - if: steps.image-exists.outputs.exists != 'true' - id: base-meta - uses: docker/metadata-action@v3 - with: - images: | - ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }} - tags: | - type=raw,value=py${{ env.PYTHON_VERSION }}-indy-${{ env.INDY_VERSION }}-nightly - type=sha,format=long,prefix=py${{ env.PYTHON_VERSION }}-indy-${{ env.INDY_VERSION }}-nightly- - - - name: Build and Push Base Image to ghcr.io - if: steps.image-exists.outputs.exists != 'true' - uses: docker/build-push-action@v3 - with: - push: true - context: . - file: docker/Dockerfile.indy - tags: ${{ steps.base-meta.outputs.tags }} - labels: ${{ steps.base-meta.outputs.labels }} - build-args: | - python_version=${{ env.PYTHON_VERSION }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - if: steps.image-exists.outputs.exists != 'true' - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache-base - diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml new file mode 100644 index 0000000000..72e51140fb --- /dev/null +++ b/.github/workflows/nightly-tests.yml @@ -0,0 +1,29 @@ +name: Nightly Tests + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +jobs: + tests: + name: Tests + strategy: + fail-fast: false + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + uses: ./.github/workflows/tests.yml + with: + python-version: ${{ matrix.python-version }} + + tests-indy: + name: Tests (Indy) + strategy: + fail-fast: false + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + + uses: ./.github/workflows/tests-indy.yml + with: + python-version: ${{ matrix.python-version }} + indy-version: "1.16.0" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 4a02ae3f1e..0000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Nightly Build -on: - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -env: - NAME: aries-cloudagent-python - PYTHON_VERSION: 3.7 - -jobs: - nightly: - name: Nightly - runs-on: ubuntu-latest - steps: - - name: Gather image info - id: info - run: | - echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" - - - name: Check image exists - id: image-exists - uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef - with: - tag: ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }}:py${{ env.PYTHON_VERSION }}-nightly-${{ github.sha }} - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/checkout@v3 - if: steps.image-exists.outputs.exists != 'true' - - - name: Cache Docker layers - if: steps.image-exists.outputs.exists != 'true' - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Set up Docker Buildx - if: steps.image-exists.outputs.exists != 'true' - uses: docker/setup-buildx-action@v1 - - - name: Log in to the GitHub Container Registry - if: steps.image-exists.outputs.exists != 'true' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Base Image Metadata - if: steps.image-exists.outputs.exists != 'true' - id: base-meta - uses: docker/metadata-action@v3 - with: - images: | - ghcr.io/${{ steps.info.outputs.repo-owner }}/${{ env.NAME }} - tags: | - type=raw,value=py${{ env.PYTHON_VERSION }}-nightly - type=sha,format=long,prefix=py${{ env.PYTHON_VERSION }}-nightly- - - - name: Build and Push Base Image to ghcr.io - if: steps.image-exists.outputs.exists != 'true' - uses: docker/build-push-action@v3 - with: - push: true - context: . - file: docker/Dockerfile - tags: ${{ steps.base-meta.outputs.tags }} - labels: ${{ steps.base-meta.outputs.labels }} - build-args: | - python_version=${{ env.PYTHON_VERSION }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - if: steps.image-exists.outputs.exists != 'true' - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache-base diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml new file mode 100644 index 0000000000..228ec7dd7f --- /dev/null +++ b/.github/workflows/pr-tests.yml @@ -0,0 +1,18 @@ +name: PR Tests + +on: + pull_request: + +jobs: + tests: + name: Tests + uses: ./.github/workflows/tests.yml + with: + python-version: "3.6" + + tests-indy: + name: Tests (Indy) + uses: ./.github/workflows/tests-indy.yml + with: + python-version: "3.6" + indy-version: "1.16.0" diff --git a/.github/workflows/publish-indy-python.yml b/.github/workflows/publish-indy-python.yml deleted file mode 100644 index f2b0bea15b..0000000000 --- a/.github/workflows/publish-indy-python.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Publish Indy Python -on: - workflow_dispatch: - inputs: - indy_sdk_url: - description: 'Indy SDK download URL' - required: false - type: string - indy_postgres_url: - description: 'Indy postgres download URL' - required: false - type: string - indy_version: - description: 'Indy SDK Version' - required: false - type: string - default: '1.16.0' - tag: - description: 'Image tag' - required: false - type: string - -env: - INDY_SDK_TAG_URL: "https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/" - -jobs: - publish-image: - strategy: - fail-fast: false - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - - name: Publish Indy Python - runs-on: ubuntu-latest - steps: - - name: Gather image info - id: info - run: | - echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" - - [ -n "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ inputs.indy_sdk_url }}" - [ -z "${{ inputs.indy_sdk_url }}"] && echo "::set-output name=indy-sdk-url::${{ env.INDY_SDK_TAG_URL }}v${{ inputs.indy_version }}" - - [ -n "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ inputs.indy_postgres_url }}" - [ -z "${{ inputs.indy_postgres_url }}"] && echo "::set-output name=indy-postgres-url::${{ env.INDY_SDK_TAG_URL }}v${{ inputs.indy_version }}" - - [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" - [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-${{ inputs.indy_version }}" - - - uses: actions/checkout@v3 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache-base - key: ${{ runner.os }}-buildx-base-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-base- - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log in to the GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Image Metadata - id: base-meta - uses: docker/metadata-action@v3 - with: - images: | - ghcr.io/${{ steps.info.outputs.repo-owner }}/indy-python - tags: | - type=raw,value=${{ steps.info.outputs.tag }} - - - name: Build and Push Image to ghcr.io - uses: docker/build-push-action@v3 - with: - push: true - context: . - file: docker/Dockerfile.indy-base - tags: ${{ steps.base-meta.outputs.tags }} - labels: ${{ steps.base-meta.outputs.labels }} - build-args: | - python_version=${{ matrix.python-version }} - indy_version=${{ inputs.indy_version }} - indy_sdk_url=${{ steps.info.outputs.indy-sdk-url }} - indy_postgres_url=${{ steps.info.outputs.indy-postgres-url }} - cache-from: type=local,src=/tmp/.buildx-cache-base - cache-to: type=local,dest=/tmp/.buildx-cache-base-new,mode=max - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache-base - mv /tmp/.buildx-cache-base-new /tmp/.buildx-cache-base diff --git a/.github/workflows/publish-indy.yml b/.github/workflows/publish-indy.yml index 058187d030..dd7c23cdc7 100644 --- a/.github/workflows/publish-indy.yml +++ b/.github/workflows/publish-indy.yml @@ -1,20 +1,24 @@ name: Publish ACA-Py (Indy) on: + release: + types: [released] workflow_dispatch: inputs: indy_version: description: 'Indy SDK Version' required: false type: string - default: '1.16.0' tag: description: 'Image tag' - required: false + required: true + type: string + version: + description: "Version label in image" + required: true type: string env: - ACAPY_REQS: '[askar,bbs]' - + INDY_VERSION: 1.16.0 jobs: publish-image: @@ -32,21 +36,14 @@ jobs: id: info run: | echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" - echo "::set-output name=acapy-version::$(sed -ne 's/__version__ = \"\([0-9.]\+\)\"/\1/p' aries_cloudagent/version.py)" - - - name: Tag image - id: tag - run: | - [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" - [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-indy-${{ inputs.indy_version }}-${{ steps.info.outputs.acapy-version }}" - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-base-${{ github.sha }} + key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx-base- + ${{ runner.os }}-buildx- - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -58,14 +55,25 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Image Metadata - id: base-meta + - name: Setup Image Metadata (manual) + if: github.event_name == 'workflow_dispatch' + id: dispatch-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python + tags: | + type=raw,value=${{ inputs.tag }} + + - name: Setup Image Metadata (release) + if: github.event_name == 'release' + id: meta uses: docker/metadata-action@v3 with: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python tags: | - type=raw,value=${{ steps.tag.outputs.tag }} + type=semver,pattern=py${{ matrix.python-version }}-indy-{{ inputs.indy_version || env.INDY_VERSION }}-{{version}} - name: Build and Push Image to ghcr.io uses: docker/build-push-action@v3 @@ -73,14 +81,12 @@ jobs: push: true context: . file: docker/Dockerfile.indy - tags: ${{ steps.base-meta.outputs.tags }} - labels: ${{ steps.base-meta.outputs.labels }} + tags: ${{ steps.dispatch-meta.outputs.tags || steps.meta.outputs.tags }} + target: main build-args: | python_version=${{ matrix.python-version }} - indy_version=${{ inputs.indy_version }} - acapy_version=${{ steps.info.outputs.acapy-version }} - acapy_reqs=${{ env.ACAPY_REQS }} - org=${{ steps.info.outputs.repo-owner }} + indy_version=${{ inputs.indy_version || env.INDY_VERSION }} + acapy_version=${{ inputs.version || github.event.release.tag_name }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 15449f4350..9ecfc8df35 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,15 +1,17 @@ name: Publish ACA-Py on: + release: + types: [released] workflow_dispatch: inputs: tag: description: 'Image tag' - required: false + required: true + type: string + version: + description: "Version label in image" + required: true type: string - -env: - ACAPY_REQS: '[askar,bbs]' - jobs: publish-image: @@ -27,21 +29,14 @@ jobs: id: info run: | echo "::set-output name=repo-owner::${GITHUB_REPOSITORY_OWNER,,}" - echo "::set-output name=acapy-version::$(sed -ne 's/__version__ = \"\([0-9.]\+\)\"/\1/p' aries_cloudagent/version.py)" - - - name: Tag image - id: tag - run: | - [ -n "${{ inputs.tag }}" ] && echo "::set-output name=tag::${{ inputs.tag }}" - [ -z "${{ inputs.tag }}" ] && echo "::set-output name=tag::py${{ matrix.python-version }}-${{ steps.info.outputs.acapy-version }}" - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-base-${{ github.sha }} + key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx-base- + ${{ runner.os }}-buildx- - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -53,14 +48,25 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Image Metadata - id: base-meta + - name: Setup Image Metadata (manual) + if: github.event_name == 'workflow_dispatch' + id: dispatch-meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python + tags: | + type=raw,value=${{ inputs.tag }} + + - name: Setup Image Metadata (release) + if: github.event_name == 'release' + id: meta uses: docker/metadata-action@v3 with: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python tags: | - type=raw,value=${{ steps.tag.outputs.tag }} + type=semver,pattern=py${{ matrix.python-version }}-{{version}} - name: Build and Push Image to ghcr.io uses: docker/build-push-action@v3 @@ -68,12 +74,11 @@ jobs: push: true context: . file: docker/Dockerfile - tags: ${{ steps.base-meta.outputs.tags }} - labels: ${{ steps.base-meta.outputs.labels }} + tags: ${{ steps.dispatch-meta.outputs.tags || steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} build-args: | python_version=${{ matrix.python-version }} - acapy_version=${{ steps.info.outputs.acapy-version }} - acapy_reqs=${{ env.ACAPY_REQS }} + acapy_version=${{ inputs.version || github.event.release.tag_name }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max diff --git a/.github/workflows/tests-indy.yml b/.github/workflows/tests-indy.yml index 86d5c38601..3037354fd4 100644 --- a/.github/workflows/tests-indy.yml +++ b/.github/workflows/tests-indy.yml @@ -1,131 +1,23 @@ name: Tests (Indy) -on: - pull_request: -env: - INDY_VERSION: 1.16.0 +on: + workflow_call: + inputs: + python-version: + required: true + type: string + indy-version: + required: true + type: string jobs: - info: - name: Gather image info - runs-on: ubuntu-latest - outputs: - repo-owner: ${{ steps.info.outputs.owner-lc }} - indy-version: ${{ steps.info.outputs.indy-version }} - indy-sdk-url: ${{ steps.info.outputs.indy-sdk-url }} - base-dep-hash: ${{ steps.info.outputs.base-hash }} - test-dep-hash: ${{ steps.info.outputs.test-hash }} - steps: - - uses: actions/checkout@v3 - - name: Gather image info - id: info - run: | - echo "::set-output name=owner-lc::${GITHUB_REPOSITORY_OWNER,,}" - echo "::set-output name=indy-version::${{env.INDY_VERSION}}" - echo "::set-output name=indy-sdk-url::https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/v${{ env.INDY_VERSION }}" - echo "::set-output name=base-hash::${{ hashFiles('docker/Dockerfile.indy-base') }}" - echo "::set-output name=test-hash::${{ hashFiles('requirements*.txt', 'docker/Dockerfile.test-indy') }}" - - base-image: - name: Publish base image - needs: info - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - - steps: - - name: Check image exists - id: image-exists - uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef - with: - tag: ghcr.io/${{ needs.info.outputs.repo-owner }}/indy-python-test:py${{ matrix.python-version }}-${{ env.INDY_VERSION }}-${{ needs.info.outputs.base-dep-hash }} - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/checkout@v3 - if: steps.image-exists.outputs.exists != 'true' - - - name: Cache Docker layers - if: steps.image-exists.outputs.exists != 'true' - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache-base - key: ${{ runner.os }}-buildx-base-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-base- - - - name: Set up Docker Buildx - if: steps.image-exists.outputs.exists != 'true' - uses: docker/setup-buildx-action@v1 - - - name: Log in to the GitHub Container Registry - if: steps.image-exists.outputs.exists != 'true' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Base Image Metadata - if: steps.image-exists.outputs.exists != 'true' - id: base-meta - uses: docker/metadata-action@v3 - with: - images: | - ghcr.io/${{ needs.info.outputs.repo-owner }}/indy-python-test - tags: | - type=raw,value=py${{ matrix.python-version }}-${{ env.INDY_VERSION }}-${{ needs.info.outputs.base-dep-hash }} - - - name: Build and Push Base Image to ghcr.io - if: steps.image-exists.outputs.exists != 'true' - uses: docker/build-push-action@v3 - with: - push: true - context: . - file: docker/Dockerfile.indy-base - tags: ${{ steps.base-meta.outputs.tags }} - labels: ${{ steps.base-meta.outputs.labels }} - build-args: | - python_version=${{ matrix.python-version }} - indy_version=${{ needs.info.outputs.indy_version }} - indy_sdk_url=${{ needs.info.outputs.indy-sdk-url }} - cache-from: type=local,src=/tmp/.buildx-cache-base - cache-to: type=local,dest=/tmp/.buildx-cache-base-new,mode=max - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - if: steps.image-exists.outputs.exists != 'true' - run: | - rm -rf /tmp/.buildx-cache-base - mv /tmp/.buildx-cache-base-new /tmp/.buildx-cache-base - - - test-image: - name: Publish test image - needs: ["info", "base-image"] + tests: + name: Test Python ${{ inputs.python-version }} on Indy ${{ inputs.indy-version }} runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - steps: - - - name: Check image exists - id: image-exists - uses: dbluhm/image-tag-exists@257851f02e3473a75719e26b5a566ea5457da4ef - with: - tag: ghcr.io/${{ needs.info.outputs.repo-owner }}/acapy-test:py${{ matrix.python-version }}-${{ needs.info.outputs.indy-version }}-${{ needs.info.outputs.test-dep-hash }} - token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v3 - if: steps.image-exists.outputs.exists != 'true' - - name: Cache Docker layers - if: steps.image-exists.outputs.exists != 'true' + - name: Cache image layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache-test @@ -134,38 +26,19 @@ jobs: ${{ runner.os }}-buildx-test- - name: Set up Docker Buildx - if: steps.image-exists.outputs.exists != 'true' uses: docker/setup-buildx-action@v1 - - name: Log in to the GitHub Container Registry - if: steps.image-exists.outputs.exists != 'true' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Test Image Metadata - if: steps.image-exists.outputs.exists != 'true' - id: test-meta - uses: docker/metadata-action@v3 - with: - images: | - ghcr.io/${{ needs.info.outputs.repo-owner }}/acapy-test - tags: | - type=raw,value=py${{ matrix.python-version }}-${{ env.INDY_VERSION }}-${{ needs.info.outputs.test-dep-hash }} - - - name: Build and Push Test Image to ghcr.io - if: steps.image-exists.outputs.exists != 'true' + - name: Build test image uses: docker/build-push-action@v3 with: - push: true + load: true context: . - file: docker/Dockerfile.test-indy - tags: ${{ steps.test-meta.outputs.tags }} - labels: ${{ steps.test-meta.outputs.labels }} + file: docker/Dockerfile.indy + target: acapy-test + tags: acapy-test:latest build-args: | - base_image=ghcr.io/${{ needs.info.outputs.repo-owner }}/indy-python-test:py${{ matrix.python-version }}-${{ needs.info.outputs.indy-version }}-${{ needs.info.outputs.base-dep-hash }} + python_version=${{ inputs.python-version }} + indy_version=${{ inputs.indy-version }} cache-from: type=local,src=/tmp/.buildx-cache-test cache-to: type=local,dest=/tmp/.buildx-cache-test-new,mode=max @@ -173,26 +46,10 @@ jobs: # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 - name: Move cache - if: steps.image-exists.outputs.exists != 'true' run: | rm -rf /tmp/.buildx-cache-test mv /tmp/.buildx-cache-test-new /tmp/.buildx-cache-test - tests: - name: Tests (Indy) - needs: - - info - - base-image - - test-image - runs-on: ubuntu-latest - container: ghcr.io/${{ needs.info.outputs.repo-owner }}/acapy-test:py${{ matrix.python-version }}-${{ needs.info.outputs.indy-version }}-${{ needs.info.outputs.test-dep-hash }} - strategy: - fail-fast: false - matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - - steps: - - uses: actions/checkout@v3 - name: Run pytest run: | - pytest + docker run --rm acapy-test:latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6205a865fc..086f026172 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,23 +1,22 @@ name: Tests on: - pull_request: + workflow_call: + inputs: + python-version: + required: true + type: string jobs: tests: - name: Tests + name: Test Python ${{ inputs.python-version }} runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ inputs.python-version }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ inputs.python-version }} cache: 'pip' cache-dependency-path: 'requirements*.txt' - name: Install dependencies diff --git a/ContainerImagesAndGithubActions.md b/ContainerImagesAndGithubActions.md index e5e8ccf277..ec4a2f32db 100644 --- a/ContainerImagesAndGithubActions.md +++ b/ContainerImagesAndGithubActions.md @@ -14,16 +14,10 @@ repository and made available through the [Github Packages Container Registry](https://ghcr.io). -## Images - -The following images are built from this project - -- `ghcr.io/hyperledger/aries-cloudagent-python` - multiple variants are built - from this project; see [Tags](#tags). -- `ghcr.io/hyperledger/indy-python` - this image is used as a base for the - ACA-Py Indy variant (see [Tags](#tags)). This may be moved to a more - appropriate project in the future. +## Image +This project builds and publishes the `ghcr.io/hyperledger/aries-cloudagent-python` image. +Multiple variants are available; see [Tags](#tags). ### Tags @@ -47,12 +41,13 @@ original BC Gov images) to the Standard image is outside of the scope of this document. The ACA-Py images built by this project are tagged to indicate which of the -above variants it is. Other tags are also generated for use by developers. +above variants it is. Other tags may also be generated for use by developers. Below is a table of all generated images and their tags: Tag | Variant | Example | Description | ------------------------|----------|--------------------------|-------------------------------------------------------------------------------------------------| +py3.6-X.Y.Z | Standard | py3.6-0.7.4 | Standard image variant built on Python 3.6 for ACA-Py version X.Y.Z | py3.7-X.Y.Z | Standard | py3.7-0.7.4 | Standard image variant built on Python 3.7 for ACA-Py version X.Y.Z | py3.8-X.Y.Z | Standard | py3.8-0.7.4 | Standard image variant built on Python 3.8 for ACA-Py version X.Y.Z | py3.9-X.Y.Z | Standard | py3.9-0.7.4 | Standard image variant built on Python 3.9 for ACA-Py version X.Y.Z | @@ -62,66 +57,10 @@ py3.8-indy-A.B.C-X.Y.Z | Indy | py3.8-indy-1.16.0-0.7.4 | Standard image v py3.9-indy-A.B.C-X.Y.Z | Indy | py3.9-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.9 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | py3.10-indy-A.B.C-X.Y.Z | Indy | py3.10-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.10 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | - -#### Indy Python - -**Image Name:** `ghcr.io/hyperledger/indy-python` - -The Indy Python image is used as a base for the Indy variant of ACA-Py. It is a -debian based image with `libindy` and the Indy SDK Python wrapper installed. - -Below is a table of all generated Indy Python images and their tags: - -Tag | Example | Description | -------------------------|--------------------------|-----------------------------------------| -py3.7-X.Y.Z | py3.7-1.16.0 | Python 3.7 with Indy SDK version X.Y.Z | -py3.8-X.Y.Z | py3.8-1.16.0 | Python 3.8 with Indy SDK version X.Y.Z | -py3.9-X.Y.Z | py3.9-1.16.0 | Python 3.9 with Indy SDK version X.Y.Z | -py3.10-X.Y.Z | py3.10-1.16.0 | Python 3.10 with Indy SDK version X.Y.Z | - - -#### Nightly - -The Github Actions will also produce Nightly builds of ACA-Py. If a nightly -build at the current hash of the repo doesn't yet exist, GHA will build a -standard and Indy ACA-Py image at midnight each day. Nightly builds are produced -only for the current "active" python version. - -Below is a table of all generated Nightly images and their tags: - -Tag | Variant | Example | Description | --------------------------------------------|----------|--------------------------------------------------------------------|---------------------------------------| -py3.7-nightly | Standard | py3.7-nightly | Standard image latest nightly | -py3.7-indy-A.B.C-nightly | Indy | py3.7-indy-1.16.0-nightly | Indy image latest nightly | -py3.7-nightly-{{ commit hash }} | Standard | py3.7-nightly-96bc6a8938f0c0e2a487a069d63bcb6c8172b320 | Standard image nightly at commit hash | -py3.7-indy-A.B.C-nightly-{{ commit hash }} | Indy | py3.7-indy-1.16.0-nightly-96bc6a8938f0c0e2a487a069d63bcb6c8172b320 | Indy image nightly at commit hash | - - -#### Testing - -The Github Actions will produce images used in CI/CD checks for Indy (Indy image -tests require `libindy` which is not available on Github runners; these tests -must be run inside of a container with `libindy`). These images are only -intended for use by these checks. - -Below is a table of all generated test images and their tags: - -Image + Tag | Description | ---------------------------------------------------------------------------------------------------------|------------------------------------| -indy-python-test:py{{python-version}}-{{indy-version}}-{{hash of indy base Dockerfile}} | Base Indy Python image for testing | -acapy-test:py{{python-version}}-{{indy-version}}-{{hash of requirements*.txt and indy test Dockerfile}} | ACA-Py test image | - -## Github Actions - -Several Github Actions are used to produce the above described images. - -**TODO:** Add descriptions of actions - -## Key Differences +### Key Image Differences There are several key differences that should be noted between the two image -variants and between the BC Gov ACA-Py images and VON images and the images -produced by this project. +variants and between the BC Gov ACA-Py images. - Standard Image - Based on slim variant of Debian @@ -151,3 +90,33 @@ produced by this project. - Askar and Indy Shared libraries built from source - Built from ACA-Py python package uploaded to PyPI - Includes Indy postgres storage plugin + +## Github Actions + +- Tests (`.github/workflows/tests.yml`) - A reusable workflow that runs tests + for the Standard ACA-Py variant for a given python version. +- Tests (Indy) (`.github/workflows/tests-indy.yml`) - A reusable workflow that + runs tests for the Indy ACA-Py variant for a given python and indy version. +- PR Tests (`.github/workflows/pr-tests.yml`) - Run on pull requests; runs tests + for the Standard and Indy ACA-Py variants for a "default" python version. + Check this workflow for the current default python and Indy versions in use. +- Nightly Tests (`.github/workflows/nightly-tests.yml`) - Run nightly; runs + tests for the Standard and Indy ACA-Py variants for all currently supported + python versions. Check this workflow for the set of currently supported + versions and Indy version(s) in use. +- Publish (`.github/workflows/publish.yml`) - Run on new release published or + when manually triggered; builds and pushes the Standard ACA-Py variant to the + Github Container Registry. +- Publish (Indy) (`.github/workflows/publish-indy.yml`) - Run on new release + published or when manually triggered; builds and pushes the Indy ACA-Py + variant to the Github Container Registry. +- Integration Tests (`.github/workflows/integrationtests.yml`) - Run on pull + requests (to the hyperledger fork only); runs BDD integration tests. +- Black Format (`.github/workflows/blackformat.yml`) - Run on pull requests; + checks formatting of files modified by the PR. +- CodeQL (`.github/workflows/codeql.yml`) - Run on pull requests; performs + CodeQL analysis. +- Python Publish (`.github/workflows/pythonpublish.yml`) - Run on release + created; publishes ACA-Py python package to PyPI. +- PIP Audit (`.github/workflows/pipaudit.yml`) - Run when manually triggered; + performs pip audit. diff --git a/docker/Dockerfile b/docker/Dockerfile index 0a40fbbe40..955479f307 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ FROM python:${python_version}-slim-buster AS main ARG uid=1001 ARG user=aries ARG acapy_version -ARG acapy_reqs +ARG acapy_reqs=[askar,bbs] ENV HOME="/home/$user" \ APP_ROOT="$HOME" \ diff --git a/docker/Dockerfile.indy b/docker/Dockerfile.indy index 9fd35fd5b6..1072998985 100644 --- a/docker/Dockerfile.indy +++ b/docker/Dockerfile.indy @@ -1,7 +1,210 @@ ARG python_version=3.6.13 +ARG rust_version=1.46 + +# This image could be replaced with an "indy" image from another repo, +# such as the indy-sdk +FROM rust:${rust_version}-slim-buster as indy-builder + +ARG user=indy +ENV HOME="/home/$user" +WORKDIR $HOME +RUN mkdir -p .local/bin .local/etc .local/lib + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + automake \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + libbz2-dev \ + libffi-dev \ + libgmp-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libsecp256k1-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libzmq3-dev \ + pkg-config \ + zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +# set to --release for smaller, optimized library +ARG indy_build_flags=--release + +ARG indy_version=1.16.0 +ARG indy_sdk_url=https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/v${indy_version} + +# make local libs and binaries accessible +ENV PATH="$HOME/.local/bin:$PATH" +ENV LIBRARY_PATH="$HOME/.local/lib:$LIBRARY_PATH" + +# Download and extract indy-sdk +RUN mkdir indy-sdk && \ + curl "${indy_sdk_url}" | tar -xz -C indy-sdk + +# Build and install indy-sdk +WORKDIR $HOME/indy-sdk +RUN cd indy-sdk*/libindy && \ + cargo build ${indy_build_flags} && \ + cp target/*/libindy.so "$HOME/.local/lib" && \ + cargo clean + +# Package python3-indy +RUN tar czvf ../python3-indy.tgz -C indy-sdk*/wrappers/python . + +# grab the latest sdk code for the postgres plug-in +WORKDIR $HOME +ARG indy_postgres_url=${indy_sdk_url} +RUN mkdir indy-postgres && \ + curl "${indy_postgres_url}" | tar -xz -C indy-postgres + +# Build and install postgres_storage plugin +WORKDIR $HOME/indy-postgres +RUN cd indy-sdk*/experimental/plugins/postgres_storage && \ + cargo build ${indy_build_flags} && \ + cp target/*/libindystrgpostgres.so "$HOME/.local/lib" && \ + cargo clean + +# Clean up SDK +WORKDIR $HOME +RUN rm -rf indy-sdk indy-postgres + + +# Indy Base Image +# This image could be replaced with an "indy-python" image from another repo, +# such as the indy-sdk +FROM python:${python_version}-slim-buster as indy-base + +ARG uid=1001 +ARG user=indy ARG indy_version -ARG org=hyperledger -FROM python:${python_version}-slim-buster AS build + +ENV HOME="/home/$user" \ + APP_ROOT="$HOME" \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + PIP_NO_CACHE_DIR=off \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + RUST_LOG=warning \ + SHELL=/bin/bash \ + SUMMARY="indy-python base image" \ + DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ + This image provides all the necessary dependencies to use the indy-sdk in python. Based on Debian Buster." + +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ + io.k8s.description="$DESCRIPTION" \ + io.k8s.display-name="indy-python $indy_version" \ + name="indy-python" \ + version="$indy_version" \ + maintainer="" + +# Add indy user +RUN useradd -U -ms /bin/bash -u $uid $user + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + bzip2 \ + curl \ + git \ + less \ + libffi6 \ + libgmp10 \ + liblzma5 \ + libncurses5 \ + libncursesw5 \ + libsecp256k1-0 \ + libzmq5 \ + net-tools \ + openssl \ + sqlite3 \ + vim-tiny \ + zlib1g && \ + rm -rf /var/lib/apt/lists/* /usr/share/doc/* + +WORKDIR $HOME + +# Copy build results +COPY --from=indy-builder --chown=$user:$user $HOME . + +RUN mkdir -p $HOME/.local/bin + +# Add local binaries and aliases to path +ENV PATH="$HOME/.local/bin:$PATH" + +# Make libraries resolvable by python +ENV LD_LIBRARY_PATH="$HOME/.local/lib:$LD_LIBRARY_PATH" +RUN echo "$HOME/.local/lib" > /etc/ld.so.conf.d/local.conf && ldconfig + +# Install python3-indy +RUN pip install --no-cache-dir python3-indy.tgz && rm python3-indy.tgz + +# - In order to drop the root user, we have to make some directories writable +# to the root group as OpenShift default security model is to run the container +# under random UID. +RUN usermod -a -G 0 $user + +# Create standard directories to allow volume mounting and set permissions +# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching +RUN mkdir -p \ + $HOME/.cache/pip/http \ + $HOME/.indy-cli/networks \ + $HOME/.indy_client/wallet \ + $HOME/.indy_client/pool \ + $HOME/.indy_client/ledger-cache \ + $HOME/ledger/sandbox/data \ + $HOME/log + +# The root group needs access the directories under $HOME/.indy_client for the container to function in OpenShift. +# Also ensure the permissions on the python 'site-packages' folder are set correctly. +RUN chown -R $user:root $HOME/.indy_client \ + && chmod -R ug+rw $HOME/log $HOME/ledger $HOME/.cache $HOME/.indy_client + +USER $user + +CMD ["bash"] + + +# ACA-Py Test +# Used to run ACA-Py unit tests with Indy +FROM indy-base as acapy-test + +USER indy + +RUN mkdir src test-reports + +WORKDIR /home/indy/src + +RUN mkdir -p test-reports && chown -R indy:indy test-reports && chmod -R ug+rw test-reports + +ADD requirements*.txt ./ + +USER root +RUN pip3 install --no-cache-dir \ + -r requirements.txt \ + -r requirements.askar.txt \ + -r requirements.bbs.txt \ + -r requirements.dev.txt + +ADD --chown=indy:root . . +USER indy + +ENTRYPOINT ["/bin/bash", "-c", "pytest \"$@\"", "--"] + +# ACA-Py Builder +# Build ACA-Py wheel using setuptools +FROM python:${python_version}-slim-buster AS acapy-builder WORKDIR /src @@ -10,12 +213,15 @@ ADD . . RUN pip install setuptools wheel RUN python setup.py sdist bdist_wheel -FROM ghcr.io/${org}/indy-python:py${python_version}-${indy_version} AS main + +# ACA-Py Indy +# Install wheel from builder and commit final image +FROM indy-base AS main ARG uid=1001 ARG user=indy ARG acapy_version -ARG acapy_reqs +ARG acapy_reqs=[askar,bbs] ENV HOME="/home/$user" \ APP_ROOT="$HOME" \ @@ -28,8 +234,8 @@ ENV HOME="/home/$user" \ SHELL=/bin/bash \ SUMMARY="aries-cloudagent image" \ DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ - This image layers the python implementation of aries-cloudagent $acapy_version. Based on indy-python, \ - this image includes indy-sdk and supporting libraries." + This image layers the python implementation of aries-cloudagent $acapy_version. \ + This image includes indy-sdk and supporting libraries." LABEL summary="$SUMMARY" \ description="$DESCRIPTION" \ @@ -47,7 +253,7 @@ RUN mkdir -p $HOME/.aries_cloudagent # Also ensure the permissions on the python 'site-packages' folder are set correctly. RUN chmod -R ug+rw $HOME/.aries_cloudagent -COPY --from=build /src/dist/aries_cloudagent*.whl . +COPY --from=acapy-builder /src/dist/aries_cloudagent*.whl . RUN pip install --no-cache-dir --find-links=. aries_cloudagent${acapy_reqs} && rm aries_cloudagent*.whl diff --git a/docker/Dockerfile.indy-base b/docker/Dockerfile.indy-base deleted file mode 100644 index fb53a59b81..0000000000 --- a/docker/Dockerfile.indy-base +++ /dev/null @@ -1,170 +0,0 @@ -ARG python_version=3.6.13 -ARG rust_version=1.46 -FROM rust:${rust_version}-slim-buster as builder - -ARG user=indy -ENV HOME="/home/$user" -WORKDIR $HOME -RUN mkdir -p .local/bin .local/etc .local/lib - -# Install environment -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - automake \ - build-essential \ - ca-certificates \ - cmake \ - curl \ - git \ - libbz2-dev \ - libffi-dev \ - libgmp-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libsecp256k1-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libzmq3-dev \ - pkg-config \ - zlib1g-dev && \ - rm -rf /var/lib/apt/lists/* - -# set to --release for smaller, optimized library -ARG indy_build_flags=--release - -ARG indy_sdk_url - -# make local libs and binaries accessible -ENV PATH="$HOME/.local/bin:$PATH" -ENV LIBRARY_PATH="$HOME/.local/lib:$LIBRARY_PATH" - -# Download and extract indy-sdk -RUN mkdir indy-sdk && \ - curl "${indy_sdk_url}" | tar -xz -C indy-sdk - -# Build and install indy-sdk -WORKDIR $HOME/indy-sdk -RUN cd indy-sdk*/libindy && \ - cargo build ${indy_build_flags} && \ - cp target/*/libindy.so "$HOME/.local/lib" && \ - cargo clean - -# Package python3-indy -RUN tar czvf ../python3-indy.tgz -C indy-sdk*/wrappers/python . - -# grab the latest sdk code for the postgres plug-in -WORKDIR $HOME -ARG indy_postgres_url=${indy_sdk_url} -RUN mkdir indy-postgres && \ - curl "${indy_postgres_url}" | tar -xz -C indy-postgres - -# Build and install postgres_storage plugin -WORKDIR $HOME/indy-postgres -RUN cd indy-sdk*/experimental/plugins/postgres_storage && \ - cargo build ${indy_build_flags} && \ - cp target/*/libindystrgpostgres.so "$HOME/.local/lib" && \ - cargo clean - -# Clean up SDK -WORKDIR $HOME -RUN rm -rf indy-sdk indy-postgres - -## Start fresh (new image) ## -FROM python:${python_version}-slim-buster - - -ARG uid=1001 -ARG user=indy -ARG indy_version - -ENV HOME="/home/$user" \ - APP_ROOT="$HOME" \ - LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 \ - PIP_NO_CACHE_DIR=off \ - PYTHONUNBUFFERED=1 \ - PYTHONIOENCODING=UTF-8 \ - RUST_LOG=warning \ - SHELL=/bin/bash \ - SUMMARY="indy-python base image" \ - DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ - This image provides all the necessary dependencies to use the indy-sdk in python. Based on Debian Buster." - -LABEL summary="$SUMMARY" \ - description="$DESCRIPTION" \ - io.k8s.description="$DESCRIPTION" \ - io.k8s.display-name="indy-python $indy_version" \ - name="indy-python" \ - version="$indy_version" \ - maintainer="" - -# Add indy user -RUN useradd -U -ms /bin/bash -u $uid $user - -# Install environment -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - bzip2 \ - curl \ - git \ - less \ - libffi6 \ - libgmp10 \ - liblzma5 \ - libncurses5 \ - libncursesw5 \ - libsecp256k1-0 \ - libzmq5 \ - net-tools \ - openssl \ - sqlite3 \ - vim-tiny \ - zlib1g && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* - -WORKDIR $HOME - -# Copy build results -COPY --from=builder --chown=$user:$user $HOME . - -RUN mkdir -p $HOME/.local/bin - -# Add local binaries and aliases to path -ENV PATH="$HOME/.local/bin:$PATH" - -# Make libraries resolvable by python -ENV LD_LIBRARY_PATH="$HOME/.local/lib:$LD_LIBRARY_PATH" -RUN echo "$HOME/.local/lib" > /etc/ld.so.conf.d/local.conf && ldconfig - -# Install python3-indy -RUN pip install --no-cache-dir python3-indy.tgz && rm python3-indy.tgz - -# - In order to drop the root user, we have to make some directories writable -# to the root group as OpenShift default security model is to run the container -# under random UID. -RUN usermod -a -G 0 $user - -# Create standard directories to allow volume mounting and set permissions -# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching -RUN mkdir -p \ - $HOME/.cache/pip/http \ - $HOME/.indy-cli/networks \ - $HOME/.indy_client/wallet \ - $HOME/.indy_client/pool \ - $HOME/.indy_client/ledger-cache \ - $HOME/ledger/sandbox/data \ - $HOME/log - -# The root group needs access the directories under $HOME/.indy_client for the container to function in OpenShift. -# Also ensure the permissions on the python 'site-packages' folder are set correctly. -RUN chown -R $user:root $HOME/.indy_client \ - && chmod -R ug+rw $HOME/log $HOME/ledger $HOME/.cache $HOME/.indy_client - -USER $user - -CMD ["bash"] diff --git a/scripts/run_tests_indy b/scripts/run_tests_indy index 6efa27b215..37fab8d5e7 100755 --- a/scripts/run_tests_indy +++ b/scripts/run_tests_indy @@ -3,7 +3,12 @@ cd "$(dirname "$0")" || exit CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-docker}" -$CONTAINER_RUNTIME build -t aries-cloudagent-test -f ../docker/Dockerfile.test-indy .. || exit 1 +DOCKER_BUILDKIT=1 $CONTAINER_RUNTIME build \ + -t aries-cloudagent-test \ + -f ../docker/Dockerfile.indy \ + --target acapy-test .. \ + --build-arg indy_version=1.16.0 \ + || exit 1 if [ ! -d ../test-reports ]; then mkdir ../test-reports; fi From 0eb63e0d7f41775eed52796c22f534f0a1f36733 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 11:45:36 -0500 Subject: [PATCH 10/17] fix: remove no longer needed files Signed-off-by: Daniel Bluhm --- docker/Dockerfile.test-indy | 26 ------------------- docker/Makefile | 51 ------------------------------------- 2 files changed, 77 deletions(-) delete mode 100644 docker/Dockerfile.test-indy delete mode 100644 docker/Makefile diff --git a/docker/Dockerfile.test-indy b/docker/Dockerfile.test-indy deleted file mode 100644 index 6b38d885e6..0000000000 --- a/docker/Dockerfile.test-indy +++ /dev/null @@ -1,26 +0,0 @@ -ARG python_version=3.6.13 -ARG indy_version=1.16.0 -ARG base_image=ghcr.io/hyperledger/indy-python:py${python_version}-${indy_version} -FROM ${base_image} - -USER indy - -RUN mkdir src test-reports - -WORKDIR /home/indy/src - -RUN mkdir -p test-reports && chown -R indy:indy test-reports && chmod -R ug+rw test-reports - -ADD requirements*.txt ./ - -USER root -RUN pip3 install --no-cache-dir \ - -r requirements.txt \ - -r requirements.askar.txt \ - -r requirements.bbs.txt \ - -r requirements.dev.txt - -ADD --chown=indy:root . . -USER indy - -ENTRYPOINT ["/bin/bash", "-c", "pytest \"$@\"", "--"] diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index c5ecf7c398..0000000000 --- a/docker/Makefile +++ /dev/null @@ -1,51 +0,0 @@ -# A simple makefile purely to demonstrate building the new docker images - -CONTAINER_RUNTIME ?= docker -IMAGE_NAME=ghcr.io/hyperledger/aries-cloudagent-python -PYTHON_VERSION=3.6.13 -PYTHON_VERSION_MAJ_MIN=3.6 -RUST_VERSION=1.46 -ACAPY_VERSION=0.7.4 -ACAPY_REQS=[askar,bbs] -INDY_VERSION=1.16.0 -INDY_SDK_URL=https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/v$(INDY_VERSION) -INDY_IMAGE_NAME=ghcr.io/hyperledger/indy-python - -all: indy-python indy standard - -indy-python: - $(CONTAINER_RUNTIME) build -t $(INDY_IMAGE_NAME):latest \ - --build-arg python_version=$(PYTHON_VERSION) \ - --build-arg rust_version=$(RUST_VERSION) \ - --build-arg indy_version=$(INDY_VERSION) \ - --build-arg indy_sdk_url=$(INDY_SDK_URL) \ - -f Dockerfile.indy-base . - $(CONTAINER_RUNTIME) tag $(INDY_IMAGE_NAME):latest \ - $(INDY_IMAGE_NAME):py$(PYTHON_VERSION)-$(INDY_VERSION) - $(CONTAINER_RUNTIME) tag $(INDY_IMAGE_NAME):latest \ - $(INDY_IMAGE_NAME):py$(PYTHON_VERSION_MAJ_MIN)-$(INDY_VERSION) - -indy: - $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):indy-latest \ - --build-arg python_version=$(PYTHON_VERSION) \ - --build-arg indy_version=$(INDY_VERSION) \ - --build-arg acapy_version=$(ACAPY_VERSION) \ - --build-arg acapy_reqs=$(ACAPY_REQS) \ - -f Dockerfile.indy . - $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):indy-latest \ - $(IMAGE_NAME):py$(PYTHON_VERSION)-indy-$(INDY_VERSION)-$(ACAPY_VERSION) - $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):indy-latest \ - $(IMAGE_NAME):py$(PYTHON_VERSION_MAJ_MIN)-indy-$(INDY_VERSION)-$(ACAPY_VERSION) - -standard: - $(CONTAINER_RUNTIME) build -t $(IMAGE_NAME):latest \ - --build-arg python_version=$(PYTHON_VERSION) \ - --build-arg acapy_version=$(ACAPY_VERSION) \ - --build-arg acapy_reqs=$(ACAPY_REQS) \ - -f Dockerfile . - $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):latest \ - $(IMAGE_NAME):py$(PYTHON_VERSION)-$(ACAPY_VERSION) - $(CONTAINER_RUNTIME) tag $(IMAGE_NAME):latest \ - $(IMAGE_NAME):py$(PYTHON_VERSION_MAJ_MIN)-$(ACAPY_VERSION) - -.PHONY: all indy-python indy standard From a90e568572bf5346ac177d483f931b699a198cb7 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 11:48:57 -0500 Subject: [PATCH 11/17] fix: paths in integrationtests worflow Signed-off-by: Daniel Bluhm --- .github/workflows/integrationtests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml index 404f7c562a..454e579ebc 100644 --- a/.github/workflows/integrationtests.yml +++ b/.github/workflows/integrationtests.yml @@ -17,9 +17,9 @@ jobs: with: path: acapy #- name: run-von-network - # uses: ./acapy/actions/run-von-network + # uses: ./acapy/.github/actions/run-von-network #- name: run-indy-tails-server - # uses: ./acapy/actions/run-indy-tails-server + # uses: ./acapy/.github/actions/run-indy-tails-server - name: run-integration-tests uses: ./acapy/.github/actions/run-integration-tests # to run with a specific set of tests include the following parameter: From 9baf15c888c9b80e44d17dc96b83bbb79d06457f Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 12:15:07 -0500 Subject: [PATCH 12/17] fix: run python 3.6 on ubuntu-20.04 Signed-off-by: Daniel Bluhm --- .github/workflows/nightly-tests.yml | 14 ++++++++++++-- .github/workflows/pr-tests.yml | 2 ++ .github/workflows/tests-indy.yml | 5 ++++- .github/workflows/tests.yml | 5 ++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 72e51140fb..e9b6d5d77a 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -11,19 +11,29 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + os: ["ubuntu-latest"] + python-version: ["3.7", "3.8", "3.9", "3.10"] + include: + - os: "ubuntu-20.04" + python-version: "3.6" uses: ./.github/workflows/tests.yml with: python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} tests-indy: name: Tests (Indy) strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + os: ["ubuntu-latest"] + python-version: ["3.7", "3.8", "3.9", "3.10"] + include: + - os: "ubuntu-20.04" + python-version: "3.6" uses: ./.github/workflows/tests-indy.yml with: python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} indy-version: "1.16.0" diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 228ec7dd7f..851ea9cf35 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -9,6 +9,7 @@ jobs: uses: ./.github/workflows/tests.yml with: python-version: "3.6" + os: "ubuntu-20.04" tests-indy: name: Tests (Indy) @@ -16,3 +17,4 @@ jobs: with: python-version: "3.6" indy-version: "1.16.0" + os: "ubuntu-20.04" diff --git a/.github/workflows/tests-indy.yml b/.github/workflows/tests-indy.yml index 3037354fd4..a893acf5b5 100644 --- a/.github/workflows/tests-indy.yml +++ b/.github/workflows/tests-indy.yml @@ -9,11 +9,14 @@ on: indy-version: required: true type: string + os: + required: true + type: string jobs: tests: name: Test Python ${{ inputs.python-version }} on Indy ${{ inputs.indy-version }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 086f026172..e098b1dd3c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,11 +6,14 @@ on: python-version: required: true type: string + os: + required: true + type: string jobs: tests: name: Test Python ${{ inputs.python-version }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 - name: Set up Python ${{ inputs.python-version }} From 024b0bef7fb0749394ba1eb362b1c717f3ebb315 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 12:17:22 -0500 Subject: [PATCH 13/17] fix: use repo owner instead of actor as username This averts issues that occur in the hyperledger org to publish values to ghcr.io Signed-off-by: Daniel Bluhm --- .github/workflows/publish-indy.yml | 2 +- .github/workflows/publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-indy.yml b/.github/workflows/publish-indy.yml index dd7c23cdc7..6fda9c50c5 100644 --- a/.github/workflows/publish-indy.yml +++ b/.github/workflows/publish-indy.yml @@ -52,7 +52,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Setup Image Metadata (manual) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9ecfc8df35..c56d4675f2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -45,7 +45,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Setup Image Metadata (manual) From a613242e85361750ac1bd3eb071af2bcf5e2b0b3 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 12:36:47 -0500 Subject: [PATCH 14/17] fix: attempt to fix tagged images Signed-off-by: Daniel Bluhm --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c56d4675f2..8c4dfc56a7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -56,7 +56,7 @@ jobs: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python tags: | - type=raw,value=${{ inputs.tag }} + type=raw,value=py${{ matrix.python-version }}-${{ inputs.tag }} - name: Setup Image Metadata (release) if: github.event_name == 'release' From 564a9af261c085670021f3913aceea7efcf96bb4 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 12:43:36 -0500 Subject: [PATCH 15/17] fix: manually tagged indy image has python-version Signed-off-by: Daniel Bluhm --- .github/workflows/publish-indy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-indy.yml b/.github/workflows/publish-indy.yml index 6fda9c50c5..084380e386 100644 --- a/.github/workflows/publish-indy.yml +++ b/.github/workflows/publish-indy.yml @@ -63,7 +63,7 @@ jobs: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python tags: | - type=raw,value=${{ inputs.tag }} + type=raw,value=py${{ matrix.python-version }}-${{ inputs.tag }} - name: Setup Image Metadata (release) if: github.event_name == 'release' From dd7310fd5a67412f95ee506d86c39fc85ba693ca Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Fri, 2 Dec 2022 12:52:19 -0500 Subject: [PATCH 16/17] fix: clarifications in docker images readme Signed-off-by: Daniel Bluhm --- ContainerImagesAndGithubActions.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ContainerImagesAndGithubActions.md b/ContainerImagesAndGithubActions.md index ec4a2f32db..25bfca9dc8 100644 --- a/ContainerImagesAndGithubActions.md +++ b/ContainerImagesAndGithubActions.md @@ -57,7 +57,7 @@ py3.8-indy-A.B.C-X.Y.Z | Indy | py3.8-indy-1.16.0-0.7.4 | Standard image v py3.9-indy-A.B.C-X.Y.Z | Indy | py3.9-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.9 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | py3.10-indy-A.B.C-X.Y.Z | Indy | py3.10-indy-1.16.0-0.7.4 | Standard image variant built on Python 3.10 for ACA-Py version X.Y.Z and Indy SDK Version A.B.C | -### Key Image Differences +### Image Comparison There are several key differences that should be noted between the two image variants and between the BC Gov ACA-Py images. @@ -67,18 +67,15 @@ variants and between the BC Gov ACA-Py images. - Does **NOT** include `libindy` - Default user is `aries` - Uses container's system python environment rather than `pyenv` - - Askar and Indy Shared libraries are installed through pip from - pre-compiled binaries included in the python wrappers. + - Askar and Indy Shared libraries are installed as dependencies of ACA-Py through pip from pre-compiled binaries included in the python wrappers - Built from repo contents - Indy Image - Based on slim variant of Debian - - Based on `indy-python` + - Built from multi-stage build step (`indy-base` in the Dockerfile) which includes Indy dependencies; this could be replaced with an explicit `indy-python` image from the Indy SDK repo - Includes `libindy` but does **NOT** include the Indy CLI - Default user is `indy` - - Based on `indy-python` - Uses container's system python environment rather than `pyenv` - - Askar and Indy Shared libraries are installed through pip from - pre-compiled binaries included in the python wrappers + - Askar and Indy Shared libraries are installed as dependencies of ACA-Py through pip from pre-compiled binaries included in the python wrappers - Built from repo contents - Includes Indy postgres storage plugin - `bcgovimages/aries-cloudagent` From 47e97bcb1715fa192433de12075da56aa6289f5f Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Mon, 19 Dec 2022 21:06:49 -0500 Subject: [PATCH 17/17] fix: metadata action in publish workflows Signed-off-by: Daniel Bluhm --- .github/workflows/publish-indy.yml | 6 +++--- .github/workflows/publish.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-indy.yml b/.github/workflows/publish-indy.yml index 084380e386..8761807d22 100644 --- a/.github/workflows/publish-indy.yml +++ b/.github/workflows/publish-indy.yml @@ -58,17 +58,17 @@ jobs: - name: Setup Image Metadata (manual) if: github.event_name == 'workflow_dispatch' id: dispatch-meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4.1.1 with: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python tags: | - type=raw,value=py${{ matrix.python-version }}-${{ inputs.tag }} + type=raw,value=py${{ matrix.python-version }}-indy-${{ inputs.tag }} - name: Setup Image Metadata (release) if: github.event_name == 'release' id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4.1.1 with: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8c4dfc56a7..b270b55a8a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,7 +51,7 @@ jobs: - name: Setup Image Metadata (manual) if: github.event_name == 'workflow_dispatch' id: dispatch-meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4.1.1 with: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python @@ -61,7 +61,7 @@ jobs: - name: Setup Image Metadata (release) if: github.event_name == 'release' id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4.1.1 with: images: | ghcr.io/${{ steps.info.outputs.repo-owner }}/aries-cloudagent-python