From c9ad3525123e78af5300803e3e0f4eceba666775 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 21 Oct 2024 12:12:55 -0700 Subject: [PATCH] Factor out tar-artifacts action --- .github/scripts/tar-artifact.sh | 107 ----------------------------- .github/workflows/kernel-build.yml | 17 +++-- tar-artifacts/action.yml | 28 ++++++++ tar-artifacts/tar-artifacts.sh | 79 +++++++++++++++++++++ 4 files changed, 119 insertions(+), 112 deletions(-) delete mode 100644 .github/scripts/tar-artifact.sh create mode 100644 tar-artifacts/action.yml create mode 100755 tar-artifacts/tar-artifacts.sh diff --git a/.github/scripts/tar-artifact.sh b/.github/scripts/tar-artifact.sh deleted file mode 100644 index 549acf9..0000000 --- a/.github/scripts/tar-artifact.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -set -eux - -# Assumptions: -# - $(pwd) is the root of kernel repo we're tarring -# - zstd is installed by default in the runner images - -if [ ! -d "${KBUILD_OUTPUT:-}" ]; then - echo "KBUILD_OUTPUT must be a directory" - exit 1 -fi - -arch="${1}" -toolchain="${2}" - -# Convert a platform (as returned by uname -m) to the kernel -# arch (as expected by ARCH= env). -platform_to_kernel_arch() { - case $1 in - s390x) - echo "s390" - ;; - aarch64) - echo "arm64" - ;; - riscv64) - echo "riscv" - ;; - x86_64) - echo "x86" - ;; - *) - echo "$1" - ;; - esac -} - -# Remove intermediate object files that we have no use for. Ideally -# we'd just exclude them from tar below, but it does not provide -# options to express the precise constraints. -find tools/testing/selftests/ -name "*.o" -a ! -name "*.bpf.o" -print0 | \ - xargs --null --max-args=10000 rm - -# Strip debug information, which is excessively large (consuming -# bandwidth) while not actually being used (the kernel does not use -# DWARF to symbolize stacktraces). -"${arch}"-linux-gnu-strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux - -image_name=$(make ARCH="$(platform_to_kernel_arch "${arch}")" -s image_name) -kbuild_output_file_list=( - ".config" - "${image_name}" - "include/config/auto.conf" - "include/generated/autoconf.h" - "vmlinux" -) - -# While we are preparing the tarball, move $KBUILD_OUTPUT to a tmp -# location just in case it's inside the repo root -tmp=$(mktemp -d) -mv "${KBUILD_OUTPUT}" "${tmp}" -stashed_kbuild_output=${tmp}/$(basename "${KBUILD_OUTPUT}") - -# Note: ${local_kbuild_output} must point to ./kbuild-output because -# of the tar command at the bottom. -local_kbuild_output=$(realpath kbuild-output) -mkdir -p "${local_kbuild_output}" - -for file in "${kbuild_output_file_list[@]}"; do - mkdir -p "$(dirname "${local_kbuild_output}/${file}")" - cp -a "${stashed_kbuild_output}/${file}" "${local_kbuild_output}/${file}" -done - -additional_file_list=() -if [[ -n "${ARCHIVE_MAKE_HELPERS}" ]]; then - # Package up a bunch of additional infrastructure to support running - # 'make kernelrelease' and bpf tool checks later on. - mapfile -t additional_file_list < <(find . -iname Makefile) - additional_file_list+=( - "scripts/" - "tools/testing/selftests/bpf/" - "tools/include/" - "tools/bpf/bpftool/" - ) -fi - -mkdir -p selftests -cp -r tools/testing/selftests/bpf selftests -if [[ -n "${BUILD_SCHED_EXT_SELFTESTS}" ]]; then - cp -r tools/testing/selftests/sched_ext selftests -fi - -tar -cf - \ - kbuild-output \ - "${additional_file_list[@]}" \ - --exclude '*.cmd' \ - --exclude '*.d' \ - --exclude '*.h' \ - --exclude '*.output' \ - selftests/ \ - | zstd -T0 -19 -o "vmlinux-${arch}-${toolchain}.tar.zst" - -# Cleanup and restore the original KBUILD_OUTPUT -# We have to put KBUILD_OUTPUT back to its original location for actions/cache -rm -rf "${local_kbuild_output}" -mv "${stashed_kbuild_output}" "${KBUILD_OUTPUT}" diff --git a/.github/workflows/kernel-build.yml b/.github/workflows/kernel-build.yml index 35c22b3..594d2a3 100644 --- a/.github/workflows/kernel-build.yml +++ b/.github/workflows/kernel-build.yml @@ -45,7 +45,6 @@ jobs: runs-on: ${{ fromJSON(inputs.runs_on) }} timeout-minutes: 100 env: - ARCHIVE_MAKE_HELPERS: ${{ github.repository != 'kernel-patches/bpf' && 'true' || '' }} KERNEL: ${{ inputs.kernel }} REPO_ROOT: ${{ github.workspace }} REPO_PATH: "" @@ -140,9 +139,17 @@ jobs: max-make-jobs: 32 llvm-version: ${{ inputs.llvm-version }} - name: Tar artifacts - working-directory: ${{ env.REPO_ROOT }} - run: | - bash .github/scripts/tar-artifact.sh ${{ inputs.arch }} ${{ inputs.toolchain_full }} + id: tar-artifacts + uses: ./tar-artifacts + env: + ARCHIVE_BPF_SELFTESTS: 'true' + ARCHIVE_MAKE_HELPERS: ${{ github.repository != 'kernel-patches/bpf' && 'true' || '' }} + ARCHIVE_SCHED_EXT_SELFTESTS: ${{ env.BUILD_SCHED_EXT_SELFTESTS }} + with: + kbuild-output: ${{ env.KBUILD_OUTPUT }} + repo-root: ${{ env.REPO_ROOT }} + arch: ${{ inputs.arch }} + toolchain: ${{ inputs.toolchain }} - if: ${{ github.event_name != 'push' }} name: Remove KBUILD_OUTPUT content shell: bash @@ -155,4 +162,4 @@ jobs: with: name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}${{ inputs.release && '-release' || '' }} if-no-files-found: error - path: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst + path: ${{ steps.tar-artifacts.outputs.archive-name }} diff --git a/tar-artifacts/action.yml b/tar-artifacts/action.yml new file mode 100644 index 0000000..f7a6d8f --- /dev/null +++ b/tar-artifacts/action.yml @@ -0,0 +1,28 @@ +name: 'Tar build artifacts' +inputs: + kbuild-output: + description: 'Path to the kernel build output' + required: true + repo-root: + description: "Path to the root of the kernel repository" + required: true + arch: + required: true + toolchain: + required: true +outputs: + archive-name: + description: 'Artifacts tarball name' + value: ${{ steps.run-script.outputs.archive-name }} + +runs: + using: "composite" + steps: + - name: Run tar-artifacts.sh + id: run-script + env: + KBUILD_OUTPUT: ${{ inputs.kbuild-output }} + REPO_ROOT: ${{ inputs.repo-root }} + shell: bash + run: + ${GITHUB_ACTION_PATH}/tar-artifacts.sh "${{ inputs.arch }}" "${{ inputs.toolchain }}" diff --git a/tar-artifacts/tar-artifacts.sh b/tar-artifacts/tar-artifacts.sh new file mode 100755 index 0000000..a207905 --- /dev/null +++ b/tar-artifacts/tar-artifacts.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +set -eux -o pipefail + +if [ ! -d "${REPO_ROOT:-}" ]; then + echo "REPO_ROOT must be a directory: ${REPO_ROOT}" + exit 1 +fi + +if [ ! -d "${KBUILD_OUTPUT:-}" ]; then + echo "KBUILD_OUTPUT must be a directory: ${KBUILD_OUTPUT}" + exit 1 +fi + +ARCHIVE_BPF_SELFTESTS="${ARCHIVE_BPF_SELFTESTS:-true}" +ARCHIVE_MAKE_HELPERS="${ARCHIVE_MAKE_HELPERS:-}" +ARCHIVE_SCHED_EXT_SELFTESTS="${ARCHIVE_SCHED_EXT_SELFTESTS:-}" + +arch="${1}" +toolchain="${2}" + +tarball="vmlinux-${arch}-${toolchain}.tar" + +source "${GITHUB_ACTION_PATH}/../helpers.sh" + +# Strip debug information, which is excessively large (consuming +# bandwidth) while not actually being used (the kernel does not use +# DWARF to symbolize stacktraces). +"${arch}"-linux-gnu-strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux + +image_name=$(make ARCH="$(platform_to_kernel_arch "${arch}")" -s image_name) +kbuild_output_file_list=( + ".config" + "${image_name}" + "include/config/auto.conf" + "include/generated/autoconf.h" + "vmlinux" +) + +tar -rf "${tarball}" -C "${KBUILD_OUTPUT}" \ + --transform "s,^,kbuild-output/," \ + "${kbuild_output_file_list[@]}" + +# In case artifacts are restored not to the kernel repo root, +# package up a bunch of additional infrastructure to support running +# 'make kernelrelease' and bpf tool checks later on. +if [[ -n "${ARCHIVE_MAKE_HELPERS}" ]]; then + find "${REPO_ROOT}" -iname Makefile -printf '%P\n' \ + | tar -rf "${tarball}" -C "${REPO_ROOT}" -T - + tar -rf "${tarball}" -C "${REPO_ROOT}" \ + --exclude '*.o' \ + --exclude '*.d' \ + "scripts/" \ + "tools/testing/selftests/bpf/" \ + "tools/include/" \ + "tools/bpf/bpftool/" +fi + +if [[ -n "${ARCHIVE_BPF_SELFTESTS}" ]]; then + # add .bpf.o files + find "${REPO_ROOT}/tools/testing/selftests/bpf" -name "*.bpf.o" -printf 'selftests/bpf/%P\n' \ + | tar -rf "${tarball}" -C "${REPO_ROOT}/tools/testing" -T - + # add other relevant files + tar -rf "${tarball}" -C "${REPO_ROOT}/tools/testing" \ + --exclude '*.cmd' \ + --exclude '*.d' \ + --exclude '*.h' \ + --exclude '*.o' \ + --exclude '*.output' \ + selftests/bpf/ +fi + +if [[ -n "${ARCHIVE_SCHED_EXT_SELFTESTS}" ]]; then + tar -rf "${tarball}" -C "${REPO_ROOT}/tools/testing" selftests/sched_ext/ +fi + +zst_tarball="vmlinux-${arch}-${toolchain}.tar.zst" +zstd -T0 -19 -i "${tarball}" -o "${zst_tarball}" +echo "archive-name=$(realpath "${zst_tarball}")" >> $GITHUB_OUTPUT