From 08b067f112963ea3226e6c711b8158a4d3cd815d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20H=C3=B6ner?= Date: Tue, 6 Aug 2024 15:48:55 +0200 Subject: [PATCH] Run ARM64 tests in CI (#94) --- .github/workflows/env/action.yml | 29 ++++++- .../workflows/unit-test-on-pull-request.yml | 85 +++++++++++-------- Makefile | 2 - support/run-tests.sh | 25 ++++-- 4 files changed, 95 insertions(+), 46 deletions(-) diff --git a/.github/workflows/env/action.yml b/.github/workflows/env/action.yml index 720484c1..ec417ffa 100644 --- a/.github/workflows/env/action.yml +++ b/.github/workflows/env/action.yml @@ -5,11 +5,34 @@ runs: steps: - name: Install dependencies shell: bash - run: sudo apt-get install -y llvm clang dwz cmake curl unzip - - name: Set up Go ${{matrix.go}} + run: | + # Ubuntu has ARM64 packages in an entirely different repo, so we can't just + # `dpkg --add-architecture arm64` here: we have to add a custom sources.list first. + sudo tee /etc/apt/sources.list.d/ubuntu.sources < /dev/null + Types: deb + URIs: http://azure.archive.ubuntu.com/ubuntu/ + Suites: noble noble-updates noble-backports + Components: main universe restricted multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + Architectures: amd64 + + Types: deb + URIs: http://azure.ports.ubuntu.com/ubuntu-ports/ + Suites: noble noble-updates noble-backports + Components: main universe restricted multiverse + Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + Architectures: arm64 + EOF + + sudo dpkg --add-architecture arm64 + sudo apt-get update -y + + sudo apt-get install -y llvm clang dwz curl unzip gcc-aarch64-linux-gnu \ + libc6-arm64-cross qemu-user-binfmt libc6:arm64 + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{matrix.go}} + go-version: stable check-latest: true cache-dependency-path: go.sum id: go diff --git a/.github/workflows/unit-test-on-pull-request.yml b/.github/workflows/unit-test-on-pull-request.yml index 0efb3826..40081816 100644 --- a/.github/workflows/unit-test-on-pull-request.yml +++ b/.github/workflows/unit-test-on-pull-request.yml @@ -8,13 +8,11 @@ on: jobs: lint: - name: Lint + name: Lint (${{ matrix.target_arch }}) runs-on: ubuntu-24.04 strategy: - fail-fast: true - max-parallel: 2 matrix: - go: ["stable"] + target_arch: [amd64, arm64] steps: - name: Clone code uses: actions/checkout@v4 @@ -26,59 +24,64 @@ jobs: make lint test: - name: Test + name: Test (${{ matrix.target_arch }}) runs-on: ubuntu-24.04 strategy: - fail-fast: true - max-parallel: 2 matrix: - go: ["stable"] + target_arch: [amd64, arm64] steps: - name: Clone code uses: actions/checkout@v4 - name: Set up environment uses: ./.github/workflows/env - - name: Build - run: | - echo $PATH - make test - name: Tests - run: | - make test + run: make test TARGET_ARCH=${{ matrix.target_arch }} build-integration-test-binaries: - name: Build integration test binaries + name: Build integration test binaries (${{ matrix.target_arch }}) runs-on: ubuntu-24.04 timeout-minutes: 10 + strategy: + matrix: + target_arch: [amd64, arm64] steps: - name: Clone code uses: actions/checkout@v4 - name: Set up environment uses: ./.github/workflows/env - name: Prepare integration test binaries for qemu tests - run: | - make integration-test-binaries + run: make integration-test-binaries TARGET_ARCH=${{ matrix.target_arch }} - name: Upload integration test binaries uses: actions/upload-artifact@v4 with: - name: integration-test-binaries + name: integration-test-binaries-${{ matrix.target_arch }} path: support/*.test integration-tests: - name: Integration tests (kernel v${{ matrix.kernel-version }}) + name: Integration tests (v${{ matrix.kernel }} ${{ matrix.target_arch }}) runs-on: ubuntu-24.04 needs: build-integration-test-binaries + timeout-minutes: 10 strategy: matrix: - kernel-version: - - "4.19.314" - - "5.4.276" - - "5.10.217" - - "5.15.159" - - "6.1.91" - - "6.6.31" - - "6.8.10" - - "6.9.1" + include: + # List of available kernels here: + # https://github.com/cilium/ci-kernels/pkgs/container/ci-kernels/versions?filters%5Bversion_type%5D=tagged + + # AMD64 + - { target_arch: amd64, kernel: 4.19.314 } + - { target_arch: amd64, kernel: 5.4.276 } + - { target_arch: amd64, kernel: 5.10.217 } + - { target_arch: amd64, kernel: 5.15.159 } + - { target_arch: amd64, kernel: 6.1.91 } + - { target_arch: amd64, kernel: 6.6.31 } + - { target_arch: amd64, kernel: 6.8.10 } + - { target_arch: amd64, kernel: 6.9.1 } + + # ARM64 (NOTE: older ARM64 kernels are not available in Cilium repos) + - { target_arch: arm64, kernel: 6.6.31 } + - { target_arch: arm64, kernel: 6.8.4 } + - { target_arch: arm64, kernel: 6.9.1 } steps: - name: Clone code uses: actions/checkout@v4 @@ -86,19 +89,29 @@ jobs: uses: ./.github/workflows/env - name: Install dependencies run: | - sudo apt-get -y install qemu-system-x86 + case "${{ matrix.target_arch }}" in + amd64) sudo apt-get -y install qemu-system-x86;; + arm64) sudo apt-get -y install qemu-system-arm;; + *) echo >&2 "bug: bad arch selected"; exit 1;; + esac go install github.com/florianl/bluebox@v0.0.1 sudo mv ~/go/bin/bluebox /usr/local/bin/. - name: Fetch integration test binaries uses: actions/download-artifact@v4 - with: { name: integration-test-binaries } - - name: Fetch precompiled kernels + with: { name: "integration-test-binaries-${{ matrix.target_arch }}" } + - name: Fetch precompiled kernel run: | install -d ci-kernels - echo "FROM ghcr.io/cilium/ci-kernels:${{ matrix.kernel-version }}" \ - | docker buildx build --quiet --pull --output="ci-kernels" - - mv ci-kernels/boot/ ci-kernels/${{ matrix.kernel-version }}/ - - name: Test on kernel ${{ matrix.kernel-version }} + echo "FROM ghcr.io/cilium/ci-kernels:${{ matrix.kernel }}" \ + | docker buildx build --platform linux/${{ matrix.target_arch }} \ + --quiet --pull --output="ci-kernels" - + mv ci-kernels/boot/ ci-kernels/${{ matrix.kernel }}/ + - name: Test on kernel ${{ matrix.kernel }} run: | chmod a+rx *.test - support/run-tests.sh ${{ matrix.kernel-version }} + case "${{ matrix.target_arch }}" in + amd64) export QEMU_ARCH=x86_64;; + arm64) export QEMU_ARCH=aarch64;; + *) echo >&2 "bug: bad arch selected"; exit 1;; + esac + support/run-tests.sh ${{ matrix.kernel }} diff --git a/Makefile b/Makefile index 634412f6..b28bd2ae 100644 --- a/Makefile +++ b/Makefile @@ -67,8 +67,6 @@ ebpf: GOLANGCI_LINT_VERSION = "v1.59.1" lint: generate - # We don't want to build the tracers here, so we stub them for linting - touch support/ebpf/tracer.ebpf.x86 go run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) version go run github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run --build-tags integration,linux --timeout 10m diff --git a/support/run-tests.sh b/support/run-tests.sh index 8a2a70f6..1ea20288 100755 --- a/support/run-tests.sh +++ b/support/run-tests.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # Test the current package under a different kernel. -# Requires qemu-system-x86_64 and bluebox to be installed. +# Requires qemu-system-$QEMU_ARCH and bluebox to be installed. set -eu set -o pipefail +qemu_arch="${QEMU_ARCH:-x86_64}" color_green=$'\033[32m' color_red=$'\033[31m' color_default=$'\033[39m' @@ -40,17 +41,31 @@ done < <(find . -name '*.test' -print0) additionalQemuArgs="" -supportKVM=$(grep -c -E 'vmx|svm' /proc/cpuinfo || echo "0") -if [ "$supportKVM" -ne 0 ]; then +supportKVM=$(grep -E 'vmx|svm' /proc/cpuinfo || true) +if [ ! "$supportKVM" ] && [ "$qemu_arch" = "$(uname -m)" ]; then additionalQemuArgs="-enable-kvm" fi +case "$qemu_arch" in + x86_64) + additionalQemuArgs+=" -append console=ttyS0" + bb_args+=(-a amd64) + ;; + aarch64) + additionalQemuArgs+=" -machine virt -cpu max" + bb_args+=(-a arm64) + ;; +esac + +if [ "$qemu_arch" = "aarch64" ]; then + additionalQemuArgs+=" -machine virt -cpu max" +fi + bluebox "${bb_args[@]}" || (echo "failed to generate initramfs"; exit 1) echo Testing on "${kernel_version}" -$sudo qemu-system-x86_64 ${additionalQemuArgs} \ +$sudo qemu-system-${qemu_arch} ${additionalQemuArgs} \ -nographic \ - -append "console=ttyS0" \ -monitor none \ -serial file:"${output}/test.log" \ -no-user-config \