From 22418e0e369ecc210e7bbbe02782078e01d7e5f9 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 22 Nov 2023 15:27:57 +0100 Subject: [PATCH] Add images for diracx/base, diracx/services-base and diracx/client-base --- .github/workflows/main.yml | 72 ++++++++++++++++++++++++++++++++++- base/Dockerfile | 22 +++++++++++ base/dirac_dependencies.yml | 29 ++++++++++++++ base/entrypoint.sh | 6 +++ base/environment.yml | 19 +++++++++ client-base/Dockerfile | 18 +++++++++ client-base/environment.yml | 13 +++++++ services-base/Dockerfile | 18 +++++++++ services-base/environment.yml | 22 +++++++++++ 9 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 base/Dockerfile create mode 100644 base/dirac_dependencies.yml create mode 100644 base/entrypoint.sh create mode 100644 base/environment.yml create mode 100644 client-base/Dockerfile create mode 100644 client-base/environment.yml create mode 100644 services-base/Dockerfile create mode 100644 services-base/environment.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4a03f9..db5daca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,8 +8,12 @@ on: branches: - main + +# https://github.com/docker/build-push-action/issues/461 +# https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 + jobs: - build: + build_old: runs-on: ubuntu-latest timeout-minutes: 30 strategy: @@ -36,3 +40,69 @@ jobs: push: ${{ github.event_name == 'push' && github.repository == 'DIRACGrid/container-images' && github.ref_name == 'main' }} tags: ghcr.io/diracgrid/diracx/${{ matrix.image-name }}:latest platforms: linux/amd64,linux/arm64 + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build base + id: build_base + uses: docker/build-push-action@v5 + with: + context: base + outputs: type=oci,dest=output-base.tar + tags: ghcr.io/diracgrid/diracx/base:latest + platforms: linux/amd64,linux/arm64 + - name: Extract base + run: | + mkdir output-base + tar -C output-base -xf output-base.tar + + - name: Build services-base + uses: docker/build-push-action@v5 + with: + context: services-base + build-contexts: | + ghcr.io/diracgrid/diracx/base=oci-layout://output-base@${{steps.build_base.outputs.digest}} + outputs: type=oci,dest=output-services-base.tar + tags: ghcr.io/diracgrid/diracx/services-base:latest + platforms: linux/amd64,linux/arm64 + - name: Extract services-base + run: | + mkdir output-services-base + tar -C output-services-base -xf output-services-base.tar + + - name: Build client-base + uses: docker/build-push-action@v5 + with: + context: client-base + build-contexts: | + ghcr.io/diracgrid/diracx/base=oci-layout://output-base@${{steps.build_base.outputs.digest}} + outputs: type=oci,dest=output-client-base.tar + tags: ghcr.io/diracgrid/diracx/client-base:latest + platforms: linux/amd64,linux/arm64 + - name: Extract client-base + run: | + mkdir output-client-base + tar -C output-client-base -xf output-client-base.tar + + - name: Push images + # if: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/container-images' && github.ref_name == 'main' }} + run: | + for image_name in base services-base client-base; do + oras cp --from-oci-layout "$PWD/output-images-${image_name}:latest" "ghcr.io/diracgrid/diracx/${image_name}:latest" + done diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000..824421d --- /dev/null +++ b/base/Dockerfile @@ -0,0 +1,22 @@ +FROM mambaorg/micromamba:latest + +# Copying in ENTRYPOINT script and environment specification +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml dirac_dependencies.yml /tmp/ +COPY --chown=$MAMBA_USER:$MAMBA_USER entrypoint.sh / +RUN chmod 755 /entrypoint.sh + +RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml --name=base && \ + micromamba install --freeze-installed --yes --file /tmp/dirac_dependencies.yml --name=base && \ + micromamba clean --all --yes --force-pkgs-dirs && \ + rm -rf /tmp/environment.yml /tmp/dirac_dependencies.yml + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +# In many clusters the container is ran as a random uid for security reasons. +# If we mark the conda directory as group 0 and give it group write permissions +# then we're still able to manage the environment from inside the container. +USER 0 +RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda +USER $MAMBA_USER + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/base/dirac_dependencies.yml b/base/dirac_dependencies.yml new file mode 100644 index 0000000..55679c3 --- /dev/null +++ b/base/dirac_dependencies.yml @@ -0,0 +1,29 @@ +# This yaml file contains the list of DIRAC dependencies that must be satisfied +# even if we won't need it. For example, gfal2 needs to be installed in order to +# be able to install DIRAC, even in the services. +# These dependencies are separated from the main environment.yml in the hope that +# we can one day remove entirely this file + +name: diracx +channels: + - diracgrid + - conda-forge + - nodefaults +dependencies: + - aiobotocore + - boto3 + - botocore + - db12 + - diraccfg + - dominate + - fts3 + - importlib-metadata + - m2crypto >=0.38.0 + - pexpect + - prompt-toolkit + - psutil + - pyasn1-modules + - pyparsing + - python-gfal2 + - pytz + - rucio-clients diff --git a/base/entrypoint.sh b/base/entrypoint.sh new file mode 100644 index 0000000..890d4de --- /dev/null +++ b/base/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +eval "$(micromamba shell hook --shell=posix)" +micromamba activate base +exec "$@" diff --git a/base/environment.yml b/base/environment.yml new file mode 100644 index 0000000..e2c2e5f --- /dev/null +++ b/base/environment.yml @@ -0,0 +1,19 @@ +name: diracx +channels: + - diracgrid + - conda-forge + - nodefaults +dependencies: + - cachetools + # Needed because coverage runs inside the image + # when doing the integration tests + - coverage + - git + - gitpython + - httpx + - pip + - pydantic =1.10.10 + - python =3.11 + - pyyaml + - requests + - urllib3 <2 diff --git a/client-base/Dockerfile b/client-base/Dockerfile new file mode 100644 index 0000000..96d623b --- /dev/null +++ b/client-base/Dockerfile @@ -0,0 +1,18 @@ +FROM ghcr.io/diracgrid/diracx/base + + +# Copying in ENTRYPOINT script and environment specification +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/ + +RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml --name=base && \ + micromamba clean --all --yes --force-pkgs-dirs && \ + rm -rf /tmp/environment.yml + + +# In many clusters the container is ran as a random uid for security reasons. +# If we mark the conda directory as group 0 and give it group write permissions +# then we're still able to manage the environment from inside the container. +USER 0 +RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda +USER $MAMBA_USER + diff --git a/client-base/environment.yml b/client-base/environment.yml new file mode 100644 index 0000000..f5e0ee3 --- /dev/null +++ b/client-base/environment.yml @@ -0,0 +1,13 @@ +name: diracx +channels: + - diracgrid + - conda-forge + - nodefaults +dependencies: + - aiohttp + - azure-core + - cachetools + - python-dotenv + - python-multipart + - rich + - typer diff --git a/services-base/Dockerfile b/services-base/Dockerfile new file mode 100644 index 0000000..96d623b --- /dev/null +++ b/services-base/Dockerfile @@ -0,0 +1,18 @@ +FROM ghcr.io/diracgrid/diracx/base + + +# Copying in ENTRYPOINT script and environment specification +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/ + +RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml --name=base && \ + micromamba clean --all --yes --force-pkgs-dirs && \ + rm -rf /tmp/environment.yml + + +# In many clusters the container is ran as a random uid for security reasons. +# If we mark the conda directory as group 0 and give it group write permissions +# then we're still able to manage the environment from inside the container. +USER 0 +RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda +USER $MAMBA_USER + diff --git a/services-base/environment.yml b/services-base/environment.yml new file mode 100644 index 0000000..8f43642 --- /dev/null +++ b/services-base/environment.yml @@ -0,0 +1,22 @@ +name: diracx +channels: + - diracgrid + - conda-forge + - nodefaults +dependencies: + - authlib + - aiomysql + - aiosqlite + - email-validator + - fastapi + - isodate + - opensearch-py + - pyjwt + - coverage + - python-dotenv + - python-jose + - python-multipart + - sqlalchemy + - uvicorn + - aiobotocore + - botocore