chore(ci): reduce copy layers (#179)

- uses RUN --mount to avoid COPY directives which create wasted layers for temp files - uses ubuntu 24.04 for newer podman/buildah versions which can use --mount - cannot inspect local container-storage due to something with the configuration of the ubuntu 24.04 builders (should be investigated)
ublue-os · Jul 25, 2024 · 7912772 · 7912772
1 parent a921a17
commit 7912772
Show file tree

Hide file tree

Showing 11 changed files with 106 additions and 100 deletions.
diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml
@@ -36,7 +36,7 @@ jobs:
 
   stream_info:
     name: Get Stream Info
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     outputs:
       fedora: ${{ fromJSON(steps.fetch.outputs.outputs).fedora}}
       image: ${{ fromJSON(steps.fetch.outputs.outputs).image}}
@@ -80,7 +80,7 @@ jobs:
 
   build_fcos:
     name: fedora-coreos
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: always() && !cancelled()
     needs: [workflow_info, stream_info]
     permissions:
@@ -142,31 +142,31 @@ jobs:
               echo "env.KERNEL_VERSION must not be empty or null"
               exit 1
           fi
-          skopeo inspect containers-storage:quay.io/fedora/fedora-coreos:${{ inputs.coreos_version }} > inspect.json
+          skopeo inspect docker://quay.io/fedora/fedora-coreos:${{ inputs.coreos_version }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled coreos image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/${{ env.KERNEL_FLAVOR }}-kernel:${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/${{ env.KERNEL_FLAVOR }}-kernel:${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled kernel-cache image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/akmods:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/akmods:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled akmods image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/akmods-nvidia:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/akmods-nvidia:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled akmods-nvidia image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/akmods-zfs:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/akmods-zfs:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled akmods-zfs image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
@@ -321,7 +321,7 @@ jobs:
 
   build_ucore:
     name: ucore
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: always() && !cancelled()
     needs: [workflow_info, stream_info]
     permissions:
@@ -400,31 +400,31 @@ jobs:
               echo "env.KERNEL_VERSION must not be empty or null"
               exit 1
           fi
-          skopeo inspect containers-storage:quay.io/fedora/fedora-coreos:${{ inputs.coreos_version }} > inspect.json
+          skopeo inspect docker://quay.io/fedora/fedora-coreos:${{ inputs.coreos_version }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled coreos image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/${{ env.KERNEL_FLAVOR }}-kernel:${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/${{ env.KERNEL_FLAVOR }}-kernel:${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled kernel-cache image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/akmods:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/akmods:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled akmods image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/akmods-nvidia:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/akmods-nvidia:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled akmods-nvidia image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"
               exit 1
           fi
-          skopeo inspect containers-storage:${{ env.IMAGE_REGISTRY }}/akmods-zfs:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
+          skopeo inspect docker://${{ env.IMAGE_REGISTRY }}/akmods-zfs:${{ env.KERNEL_FLAVOR }}-${{ env.FEDORA_VERSION }} > inspect.json
           kernel=$(jq -r '.["Labels"]["ostree.linux"]' inspect.json)
           if [[ "${{ env.KERNEL_VERSION }}" != "$kernel"*  ]]; then
               echo "pulled akmods-zfs image kernel ($kernel) does not match expected kernel (${{ env.KERNEL_VERSION }})"

diff --git a/fedora-coreos/Containerfile b/fedora-coreos/Containerfile
@@ -15,7 +15,9 @@ FROM ${AKMODS_ZFS} AS akmods-zfs
 FROM ${CONFIG} AS config
 FROM ${KERNEL} AS kernel
 
-# image base
+FROM scratch AS ctx
+COPY / /
+
 FROM quay.io/fedora/fedora-coreos:${COREOS_VERSION}
 
 ARG COREOS_VERSION="${COREOS_VERSION:-stable}"
@@ -24,21 +26,20 @@ ARG NVIDIA_TAG="${NVIDIA_TAG}"
 # build with --build-arg ZFS_TAG="-zfs" to install zfs
 ARG ZFS_TAG="${ZFS_TAG}"
 
-COPY --from=akmods-common /rpms/ucore/ublue*.rpm /tmp/rpms/
-COPY --from=akmods-nvidia /rpms/kmods/*.rpm /tmp/rpms/nvidia/
-COPY --from=akmods-nvidia /rpms/ucore/ublue*.rpm /tmp/rpms/nvidia/
-COPY --from=akmods-zfs /rpms/kmods/zfs/*.rpm /tmp/rpms/zfs/
-COPY --from=config /rpms/ublue-os-signing*.rpm /tmp/rpms/
-COPY --from=kernel /tmp/rpms/ /tmp/kernel-rpms/
-
-COPY *.sh /tmp/
 
-RUN mkdir -p /var/lib/alternatives \
-    && /tmp/install.sh \
-    && /tmp/post-install.sh \
+RUN --mount=type=cache,dst=/var/cache/rpm-ostree \
+    --mount=type=bind,from=ctx,src=/,dst=/ctx \
+    --mount=type=bind,from=akmods-common,src=/rpms/ucore,dst=/tmp/rpms/akmods-common \
+    --mount=type=bind,from=akmods-nvidia,src=/rpms,dst=/tmp/rpms/akmods-nvidia \
+    --mount=type=bind,from=akmods-zfs,src=/rpms,dst=/tmp/rpms/akmods-zfs \
+    --mount=type=bind,from=config,src=/rpms,dst=/tmp/rpms/config \
+    --mount=type=bind,from=kernel,src=/tmp/rpms,dst=/tmp/rpms/kernel \
+    mkdir -p /var/lib/alternatives \
+    && /ctx/install.sh \
+    && /ctx/post-install.sh \
     && mv /var/lib/alternatives /staged-alternatives \
-    && rm -fr /tmp/* /var/* \
+    && /ctx/cleanup.sh \
     && ostree container commit \
     && mkdir -p /var/lib && mv /staged-alternatives /var/lib/alternatives \
-    && mkdir -p /tmp /var/tmp \
-    && chmod -R 1777 /tmp /var/tmp
+    && mkdir -p /var/tmp \
+    && chmod -R 1777 /var/tmp
diff --git a/fedora-coreos/cleanup.sh b/fedora-coreos/cleanup.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+set -eoux pipefail
+shopt -s extglob
+
+rm -rf /tmp/* || true
+rm -rf /var/!(cache)
+rm -rf /var/cache/!(rpm-ostree)
diff --git a/fedora-coreos/install.sh b/fedora-coreos/install.sh
@@ -4,7 +4,7 @@ set -ouex pipefail
 
 ARCH="$(rpm -E %{_arch})"
 RELEASE="$(rpm -E %fedora)"
-pushd /tmp/kernel-rpms
+pushd /tmp/rpms/kernel
 KERNEL_VERSION=$(find kernel-*.rpm | grep -P "kernel-(\d+\.\d+\.\d+)-.*\.fc${RELEASE}\.${ARCH}" | sed -E 's/kernel-//' | sed -E 's/\.rpm//')
 popd
 QUALIFIED_KERNEL="$(rpm -qa | grep -P 'kernel-(\d+\.\d+\.\d+)' | sed -E 's/kernel-//')"
@@ -26,31 +26,32 @@ sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-cisco-openh264.repo
 #### INSTALL
 # inspect to see what RPMS we copied in
 find /tmp/rpms/
-find /tmp/kernel-rpms/
 
-rpm-ostree install /tmp/rpms/*.rpm
+rpm-ostree install \
+    /tmp/rpms/akmods-common/ublue-os-ucore-addons*.rpm \
+    /tmp/rpms/config/ublue-os-signing*.rpm
 
 # Handle Kernel Skew with override replace
 rpm-ostree cliwrap install-to-root /
 if [[ "${KERNEL_VERSION}" == "${QUALIFIED_KERNEL}" ]]; then
     echo "Installing signed kernel from kernel-cache."
     cd /tmp
-    rpm2cpio /tmp/kernel-rpms/kernel-core-*.rpm | cpio -idmv
+    rpm2cpio /tmp/rpms/kernel/kernel-core-*.rpm | cpio -idmv
     cp ./lib/modules/*/vmlinuz /usr/lib/modules/*/vmlinuz
     cd /
 else
     echo "Install kernel version ${KERNEL_VERSION} from kernel-cache."
     rpm-ostree override replace \
         --experimental \
         --install=zstd \
-        /tmp/kernel-rpms/kernel-[0-9]*.rpm \
-        /tmp/kernel-rpms/kernel-core-*.rpm \
-        /tmp/kernel-rpms/kernel-modules-*.rpm
+        /tmp/rpms/kernel/kernel-[0-9]*.rpm \
+        /tmp/rpms/kernel/kernel-core-*.rpm \
+        /tmp/rpms/kernel/kernel-modules-*.rpm
 fi
 
 ## CONDITIONAL: install ZFS
 if [[ "-zfs" == "${ZFS_TAG}" ]]; then
-    rpm-ostree install pv /tmp/rpms/zfs/*.rpm
+    rpm-ostree install pv /tmp/rpms/akmods-zfs/kmods/zfs/*.rpm
     # for some reason depmod ran automatically with zfs 2.1 but not with 2.2
     depmod -A ${KERNEL_VERSION}
 fi
@@ -60,11 +61,11 @@ if [[ "-nvidia" == "${NVIDIA_TAG}" ]]; then
     # repo for nvidia rpms
     curl -L https://negativo17.org/repos/fedora-nvidia.repo -o /etc/yum.repos.d/fedora-nvidia.repo
 
-    rpm-ostree install /tmp/rpms/nvidia/ublue-os-ucore-nvidia-*.rpm
+    rpm-ostree install /tmp/rpms/akmods-nvidia/ucore/ublue-os-ucore-nvidia*.rpm
     sed -i '0,/enabled=0/{s/enabled=0/enabled=1/}' /etc/yum.repos.d/nvidia-container-toolkit.repo
 
     rpm-ostree install \
-        /tmp/rpms/nvidia/kmod-nvidia-*.rpm \
+        /tmp/rpms/akmods-nvidia/kmods/kmod-nvidia*.rpm \
         nvidia-driver-cuda \
         nvidia-container-toolkit
 fi
diff --git a/ucore/Containerfile b/ucore/Containerfile
@@ -15,6 +15,9 @@ FROM ${AKMODS_ZFS} AS akmods-zfs
 FROM ${CONFIG} AS config
 FROM ${KERNEL} AS kernel
 
+FROM scratch AS ctx
+COPY / /
+
 # ucore-minimal image section
 FROM quay.io/fedora/fedora-coreos:${COREOS_VERSION} AS ucore-minimal
 
@@ -29,39 +32,26 @@ ARG DOCKER_BUILDX_VERSION=0.12.1
 # 2.24.7 matches docker/moby 24.0.5  which FCOS ships as of 40.20240421
 ARG DOCKER_COMPOSE_VERSION=v2.24.7
 
-COPY --from=akmods-common /rpms/ucore/ublue*.rpm /tmp/rpms/
-COPY --from=akmods-nvidia /rpms/kmods/*.rpm /tmp/rpms/nvidia/
-COPY --from=akmods-nvidia /rpms/ucore/ublue*.rpm /tmp/rpms/nvidia/
-COPY --from=akmods-zfs /rpms/kmods/zfs/*.rpm /tmp/rpms/zfs/
-COPY --from=config /rpms/ublue-os-signing*.rpm /tmp/rpms/
-COPY --from=kernel /tmp/rpms/ /tmp/kernel-rpms/
-
-COPY *.sh /tmp/
-COPY packages.json /tmp/packages.json
-
 COPY usr /usr
-
-RUN mkdir -p /var/lib/alternatives \
-    && /tmp/install-ucore-minimal.sh \
-    && /tmp/post-install-ucore-minimal.sh \
-    && mv /var/lib/alternatives /staged-alternatives \
-    && rm -fr /tmp/* /var/* \
-    && ostree container commit \
-    && mkdir -p /var/lib && mv /staged-alternatives /var/lib/alternatives \
-    && mkdir -p /tmp /var/tmp \
-    && chmod -R 1777 /tmp /var/tmp
-
 COPY --from=docker.io/docker/buildx-bin:${DOCKER_BUILDX_VERSION} /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 COPY --from=docker.io/docker/compose-bin:${DOCKER_COMPOSE_VERSION} /docker-compose /usr/libexec/docker/cli-plugins/docker-compose
 
-RUN ln -s /usr/libexec/docker/cli-plugins/docker-compose /usr/bin/docker-compose \
+RUN --mount=type=cache,dst=/var/cache/rpm-ostree \
+    --mount=type=bind,from=ctx,src=/,dst=/ctx \
+    --mount=type=bind,from=akmods-common,src=/rpms/ucore,dst=/tmp/rpms/akmods-common \
+    --mount=type=bind,from=akmods-nvidia,src=/rpms,dst=/tmp/rpms/akmods-nvidia \
+    --mount=type=bind,from=akmods-zfs,src=/rpms,dst=/tmp/rpms/akmods-zfs \
+    --mount=type=bind,from=config,src=/rpms,dst=/tmp/rpms/config \
+    --mount=type=bind,from=kernel,src=/tmp/rpms,dst=/tmp/rpms/kernel \
+    mkdir -p /var/lib/alternatives \
+    && /ctx/install-ucore-minimal.sh \
+    && /ctx/post-install-ucore-minimal.sh \
     && mv /var/lib/alternatives /staged-alternatives \
-    && rm -fr /tmp/* /var/* \
+    && /ctx/cleanup.sh \
     && ostree container commit \
     && mkdir -p /var/lib && mv /staged-alternatives /var/lib/alternatives \
-    && mkdir -p /tmp /var/tmp \
-    && chmod -R 1777 /tmp /var/tmp
-
+    && mkdir -p /var/tmp \
+    && chmod -R 1777 /var/tmp
 
 # ucore image section
 FROM ucore-minimal AS ucore
@@ -70,19 +60,16 @@ ARG COREOS_VERSION="${COREOS_VERSION:-stable}"
 ARG NVIDIA_TAG="${NVIDIA_TAG}"
 ARG ZFS_TAG="${ZFS_TAG}"
 
-COPY *.sh /tmp/
-COPY packages.json /tmp/packages.json
-
-COPY usr /usr
-
-RUN mkdir -p /var/lib/alternatives \
-    && /tmp/install-ucore.sh \
+RUN --mount=type=cache,dst=/var/cache/rpm-ostree \
+    --mount=type=bind,from=ctx,src=/,dst=/ctx \
+    mkdir -p /var/lib/alternatives \
+    && /ctx/install-ucore.sh \
     && mv /var/lib/alternatives /staged-alternatives \
-    && rm -fr /tmp/* /var/* \
+    && /ctx/cleanup.sh \
     && ostree container commit \
     && mkdir -p /var/lib && mv /staged-alternatives /var/lib/alternatives \
-    && mkdir -p /tmp /var/tmp \
-    && chmod -R 1777 /tmp /var/tmp
+    && mkdir -p /var/tmp \
+    && chmod -R 1777 /var/tmp
 
 
 # ucore-hci image section
@@ -92,14 +79,13 @@ ARG COREOS_VERSION="${COREOS_VERSION:-stable}"
 ARG NVIDIA_TAG="${NVIDIA_TAG}"
 ARG ZFS_TAG="${ZFS_TAG}"
 
-COPY *.sh /tmp/
-COPY packages.json /tmp/packages.json
-
-RUN mkdir -p /var/lib/alternatives \
-    && /tmp/install-ucore-hci.sh \
+RUN --mount=type=cache,dst=/var/cache/rpm-ostree \
+    --mount=type=bind,from=ctx,src=/,dst=/ctx \
+    mkdir -p /var/lib/alternatives \
+    && /ctx/install-ucore-hci.sh \
     && mv /var/lib/alternatives /staged-alternatives \
-    && rm -fr /tmp/* /var/* \
+    && /ctx/cleanup.sh \
     && ostree container commit \
     && mkdir -p /var/lib && mv /staged-alternatives /var/lib/alternatives \
-    && mkdir -p /tmp /var/tmp \
-    && chmod -R 1777 /tmp /var/tmp
+    && mkdir -p /var/tmp \
+    && chmod -R 1777 /var/tmp
diff --git a/ucore/cleanup.sh b/ucore/cleanup.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+set -eoux pipefail
+shopt -s extglob
+
+rm -rf /tmp/* || true
+rm -rf /var/!(cache)
+rm -rf /var/cache/!(rpm-ostree)
diff --git a/ucore/install-ucore-hci.sh b/ucore/install-ucore-hci.sh
@@ -4,7 +4,7 @@ set -ouex pipefail
 
 # install packages.json stuffs
 export IMAGE_NAME=ucore-hci
-/tmp/packages.sh
+/ctx/packages.sh
 
 # tweak os-release
 sed -i '/^PRETTY_NAME/s/(uCore.*$/(uCore HCI)"/' /usr/lib/os-release