testing: Import reusable workflows from kernel-patches/vmtests to tes…

…t libbpf/ci changes

Signed-off-by: Manu Bretelle <[email protected]>

.github/scripts/get-commit-metadata.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+branch="${GITHUB_BASE_REF}"
+
+if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then
+  branch="${GITHUB_REF_NAME}"
+fi
+
+echo "branch=${branch}" >> "${GITHUB_OUTPUT}"
+
+upstream="${branch//_base/}"
+commit="$(
+  git rev-parse "origin/${upstream}" &> /dev/null \
+    || (
+      git fetch --quiet --prune --no-tags --depth=1 --no-recurse-submodules origin "+refs/heads/${upstream}:refs/remotes/origin/${upstream}" && \
+      git rev-parse "origin/${upstream}"
+    )
+)"
+timestamp_utc="$(TZ=utc git show --format='%cd' --no-patch --date=iso-strict-local "${commit}")"
+
+echo "timestamp=${timestamp_utc}" >> "${GITHUB_OUTPUT}"
+echo "commit=${commit}" >> "${GITHUB_OUTPUT}"
+echo "Most recent upstream commit is ${commit}"

.github/workflows/kernel-build-test.yml

@@ -0,0 +1,98 @@
+name: Reusable Build/Test/Veristat workflow
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+        description: The architecture to build against, e.g x86_64, aarch64, s390x...
+      toolchain_full:
+        required: true
+        type: string
+        description: The toolchain and for llvm, its version, e.g gcc, llvm-15
+      toolchain:
+        required: true
+        type: string
+        description: The toolchain, e.g gcc, llvm
+      runs_on:
+        required: true
+        type: string
+        description: The runners to run the test on. This is a json string representing an array of labels.
+      build_runs_on:
+        required: true
+        type: string
+        description: The runners to run the builds on. This is a json string representing an array of labels.
+      llvm-version:
+        required: true
+        type: string
+        description: The version of LLVM used to build selftest.... for llvm toolchain, this should match the one from toolchain_full, for gcc it is an arbritrary version we decide to build selftests against.
+      kernel:
+        required: true
+        type: string
+        description: The kernel to run the test against. For KPD this is always LATEST, which runs against a newly built kernel.
+      tests:
+        required: true
+        type: string
+        description: A serialized json array with the tests to be running, it must follow the json-matrix format, https://www.jitsejan.com/use-github-actions-with-json-file-as-matrix
+      run_veristat:
+        required: true
+        type: boolean
+        description: Whether or not to run the veristat job.
+      run_tests:
+        required: true
+        type: boolean
+        description: Whether or not to run the test job.
+      download_sources:
+        required: true
+        type: boolean
+        description: Whether to download the linux sources into the working directory.
+        default: false
+      build_release:
+        required: true
+        type: boolean
+        description: Build selftests with -O2 optimization in addition to non-optimized build.
+        default: false
+
+jobs:
+  # Build kernel and selftest
+  build:
+    uses: ./.github/workflows/kernel-build.yml
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      toolchain: ${{ inputs.toolchain }}
+      runs_on: ${{ inputs.build_runs_on }}
+      llvm-version: ${{ inputs.llvm-version }}
+      kernel: ${{ inputs.kernel }}
+      download_sources: ${{ inputs.download_sources }}
+  build-release:
+    if: ${{ inputs.build_release }}
+    uses: ./.github/workflows/kernel-build.yml
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      toolchain: ${{ inputs.toolchain }}
+      runs_on: ${{ inputs.runs_on }}
+      llvm-version: ${{ inputs.llvm-version }}
+      kernel: ${{ inputs.kernel }}
+      download_sources: ${{ inputs.download_sources }}
+      release: true
+  test:
+    if: ${{ inputs.run_tests }}
+    uses: ./.github/workflows/kernel-test.yml
+    # Setting name to test here to avoid lengthy autogenerated names due to matrix
+    # e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
+    name: "test"
+    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.tests) }}
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      runs_on: ${{ inputs.runs_on }}
+      kernel: ${{ inputs.kernel }}
+      test: ${{ matrix.test }}
+      continue_on_error: ${{ toJSON(matrix.continue_on_error) }}
+      timeout_minutes: ${{ matrix.timeout_minutes }}

.github/workflows/kernel-build.yml

@@ -0,0 +1,150 @@
+
+name: Reusable build workflow
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+        description: The architecture to build against, e.g x86_64, aarch64, s390x...
+      toolchain_full:
+        required: true
+        type: string
+        description: The toolchain and for llvm, its version, e.g gcc, llvm-15
+      toolchain:
+        required: true
+        type: string
+        description: The toolchain, e.g gcc, llvm
+      runs_on:
+        required: true
+        type: string
+        description: The runners to run the test on. This is a json string representing an array of labels.
+      llvm-version:
+        required: true
+        type: string
+        description: The version of LLVM used to build selftest.... for llvm toolchain, this should match the one from toolchain_full, for gcc it is an arbritrary version we decide to build selftests against.
+      kernel:
+        required: true
+        type: string
+        description: The kernel to run the test against. For KPD this is always LATEST, which runs against a newly built kernel.
+      download_sources:
+        required: true
+        type: boolean
+        description: Whether to download the linux sources into the working directory.
+        default: false
+      release:
+        required: false
+        type: boolean
+        description: Build selftest with -O2 optimization
+        default: false
+
+jobs:
+  build:
+    name: build for ${{ inputs.arch }} with ${{ inputs.toolchain_full }}${{ inputs.release && '-O2' || '' }}
+    runs-on: ${{ fromJSON(inputs.runs_on) }}
+    timeout-minutes: 100
+    env:
+        KERNEL: ${{ inputs.kernel }}
+        REPO_ROOT: ${{ github.workspace }}
+        REPO_PATH: ""
+        KBUILD_OUTPUT: kbuild-output/
+    steps:
+      - uses: actions/checkout@v4
+        # We fetch an actual bit of history here to facilitate incremental
+        # builds (which may check out some earlier upstream change).
+        with:
+          fetch-depth: 50
+      - if: ${{ inputs.download_sources }}
+        name: Download bpf-next tree
+        uses: libbpf/ci/get-linux-source@main
+        with:
+          dest: '.kernel'
+      - if: ${{ inputs.download_sources }}
+        name: Move linux source in place
+        shell: bash
+        run: |
+          rm -rf .kernel/.git
+          cp -rf .kernel/. .
+          rm -rf .kernel
+      - name: Get commit meta-data
+        id: get-commit-metadata
+        run: |
+          bash .github/scripts/get-commit-metadata.sh
+      - name: Pull recent KBUILD_OUTPUT contents
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.KBUILD_OUTPUT }}
+          key: kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-${{ steps.get-commit-metadata.outputs.commit }}
+          restore-keys: |
+            kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-
+            kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-
+            kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-
+      - name: Prepare incremental build
+        shell: bash
+        run: |
+          bash .github/scripts/prepare-incremental-builds.sh ${{ steps.get-commit-metadata.outputs.commit }}
+      - uses: libbpf/ci/patch-kernel@main
+        with:
+          patches-root: '${{ github.workspace }}/ci/diffs'
+          repo-root: '${{ github.workspace }}'
+      - name: Setup build environment
+        uses: libbpf/ci/setup-build-env@main
+        with:
+          arch: ${{ inputs.arch }}
+          llvm-version: ${{ inputs.llvm-version }}
+          pahole: c2f89dab3f2b0ebb53bab3ed8be32f41cb743c37
+      - name: Print toolchain version used
+        shell: bash
+        run: |
+          TOOLCHAIN=${{ inputs.toolchain }}
+          if [ $TOOLCHAIN = "llvm" ]; then
+            TOOLCHAIN="clang-${{ inputs.llvm-version }}"
+          fi
+          ${TOOLCHAIN} --version
+      - name: Build kernel image
+        uses: libbpf/ci/build-linux@main
+        with:
+          arch: ${{ inputs.arch }}
+          toolchain: ${{ inputs.toolchain }}
+          kbuild-output: ${{ env.KBUILD_OUTPUT }}
+          max-make-jobs: 32
+          llvm-version: ${{ inputs.llvm-version }}
+      - name: Build selftests
+        uses: libbpf/ci/build-selftests@main
+        with:
+          arch: ${{ inputs.arch }}
+          toolchain: ${{ inputs.toolchain }}
+          kbuild-output: ${{ env.KBUILD_OUTPUT }}
+          max-make-jobs: 32
+          llvm-version: ${{ inputs.llvm-version }}
+        env:
+          # RELEASE= disables all optimizaions
+          # RELEASE=0 adds -O0 make flag
+          # RELEASE=1 adds -O2 make flag
+          RELEASE: ${{ inputs.release && '1' || '' }}
+      - if: ${{ github.event_name != 'push' }}
+        name: Build samples
+        uses: libbpf/ci/build-samples@main
+        with:
+          arch: ${{ inputs.arch }}
+          toolchain: ${{ inputs.toolchain }}
+          kbuild-output: ${{ env.KBUILD_OUTPUT }}
+          max-make-jobs: 32
+          llvm-version: ${{ inputs.llvm-version }}
+      - name: Tar artifacts
+        run: |
+          bash .github/scripts/tar-artifact.sh ${{ inputs.arch }} ${{ inputs.toolchain_full }}
+      - if: ${{ github.event_name != 'push' }}
+        name: Remove KBUILD_OUTPUT content
+        shell: bash
+        run: |
+          # Remove $KBUILD_OUTPUT to prevent cache creation for pull requests.
+          # Only on pushed changes are build artifacts actually cached, because
+          # of github.com/actions/cache's cache isolation logic.
+          rm -rf "${KBUILD_OUTPUT}"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}${{ inputs.release && '-release' || '' }}
+          if-no-files-found: error
+          path: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst

.github/workflows/kernel-test.yml

@@ -0,0 +1,69 @@
+name: Reusable test workflow
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+        description: The architecture to build against, e.g x86_64, aarch64, s390x...
+      toolchain_full:
+        required: true
+        type: string
+        description: The toolchain and for llvm, its version, e.g gcc, llvm-15
+      runs_on:
+        required: true
+        type: string
+        description: The runners to run the test on. This is a json string representing an array of labels.
+      kernel:
+        required: true
+        type: string
+        description: The kernel to run the test against. For KPD this is always LATEST, which runs against a newly built kernel.
+      test:
+        required: true
+        type: string
+        description: The test to run in the vm, e.g test_progs, test_maps, test_progs_no_alu32...
+      continue_on_error:
+        required: true
+        type: string
+        description: Whether to continue on error. This is typically set to true for parallel tests which are currently known to fail, but we don't want to fail the whole CI because of that.
+      timeout_minutes:
+        required: true
+        type: number
+        description: In case a test runs for too long, after how many seconds shall we timeout and error.
+
+jobs:
+  test:
+    name: ${{ inputs.test }} on ${{ inputs.arch }} with ${{ inputs.toolchain_full }}
+    runs-on: ${{ fromJSON(inputs.runs_on) }}
+    timeout-minutes: 100
+    env:
+      KERNEL: ${{ inputs.kernel }}
+      REPO_ROOT: ${{ github.workspace }}
+      REPO_PATH: ""
+      KBUILD_OUTPUT: kbuild-output/
+      # https://github.com/actions/runner/issues/1483#issuecomment-1031671517
+      # booleans are weird in GH.
+      CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
+          path: .
+      - name: Untar artifacts
+        # zstd is installed by default in the runner images.
+        run: zstd -d -T0  vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst --stdout | tar -xf -
+      - name: Run selftests
+        uses: libbpf/ci/run-vmtest@main
+        # https://github.com/actions/runner/issues/1483#issuecomment-1031671517
+        # booleans are weird in GH.
+        continue-on-error: ${{ fromJSON(env.CONTINUE_ON_ERROR) }}
+        timeout-minutes: ${{ inputs.timeout_minutes }}
+        with:
+          arch: ${{ inputs.arch}}
+          img: '/tmp/root.img'
+          vmlinuz: '${{ github.workspace }}/vmlinuz'
+          kernel-root: '.'
+          max-cpu: 8
+          kernel-test: ${{ inputs.test }}

.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: bpf-ci
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ci-test-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-test:
+    strategy:
+      matrix:
+        runs_on: ["ubuntu-24.04"]
+        build_runs_on: ["ubuntu-24.04"]
+        arch: ["x86_64"]
+        toolchain: [{"name": "gcc", "fullname": "gcc", "version": 17}]
+        tests: [{"include": [{"test": "test_progs", "continue_on_error": false, "timeout_minutes": 360}, {"test": "test_progs_no_alu32", "continue_on_error": false, "timeout_minutes": 360}, {"test": "test_verifier", "continue_on_error": false, "timeout_minutes": 360}, {"test": "test_maps", "continue_on_error": false, "timeout_minutes": 360}]}]
+      fail-fast: false
+    # Setting name to arch-compiler here to avoid lengthy autogenerated names due to matrix
+    # e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
+    name: "${{ matrix.arch }}-${{ matrix.toolchain.fullname }}"
+    uses: ./.github/workflows/kernel-build-test.yml
+    permissions:
+      id-token: write
+      contents: read
+
+    with:
+      arch: ${{ matrix.arch }}
+      toolchain_full: ${{ matrix.toolchain.fullname }}
+      toolchain: ${{ matrix.toolchain.name }}
+      runs_on: ${{ toJSON(matrix.runs_on) }}
+      build_runs_on: ${{ toJSON(matrix.build_runs_on) }}
+      llvm-version: ${{ matrix.toolchain.version }}
+      kernel: "LATEST"
+      tests: ${{ toJSON(matrix.tests) }}
+      # We only run tests on pull requests.
+      run_tests: ${{ github.event_name != 'push' }}
+      # Download sources
+      download_sources: true
+      build_release: false