[Release-1.32] Jan 2025 Testing Overhaul, E2E to Docker Migration,

.github/workflows/build-k3s.yaml

@@ -3,6 +3,10 @@ name: Build K3s
 on: 
   workflow_call:
    inputs:
+    arch:
+      type: string
+      description: 'Architecture to build'
+      default: 'ubuntu-latest'
     upload-repo:
       type: boolean
       required: false
@@ -18,7 +22,7 @@ permissions:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.arch }} # defaults to ubuntu-latest, for arm64 use ubuntu-24.04-arm
     timeout-minutes: 20
     steps:
     - name: Checkout K3s
@@ -44,9 +48,15 @@ jobs:
     - name: "Save K3s image"
       if: inputs.upload-image == true
       run: docker image save rancher/k3s -o ./dist/artifacts/k3s-image.tar
-    - name: "Upload K3s binary"
-      if: inputs.upload-repo == false
+    - name: "Upload K3s Artifacts"
+      if: inputs.upload-repo == false && inputs.arch == 'ubuntu-latest'
       uses: actions/upload-artifact@v4
       with:
         name: k3s
+        path: dist/artifacts/k3s*
+    - name: "Upload K3s arm64 Artifacts"
+      if: contains(inputs.arch, 'arm')
+      uses: actions/upload-artifact@v4
+      with:
+        name: k3s-arm64
         path: dist/artifacts/k3s*

.github/workflows/e2e.yaml

@@ -32,6 +32,11 @@ jobs:
     uses: ./.github/workflows/build-k3s.yaml
     with:
       upload-image: true
+  build-arm64:
+    uses: ./.github/workflows/build-k3s.yaml
+    with:
+      arch: ubuntu-24.04-arm
+      upload-image: true
   e2e:
     name: "E2E Tests"
     needs: build
@@ -100,40 +105,15 @@ jobs:
           files: tests/e2e/${{ matrix.etest }}/coverage.out
           flags: e2etests # optional
           verbose: true # optional (default = false)
-  docker:
-    needs: build
-    name: Docker Tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    strategy:
-      fail-fast: false
-      matrix:
-        dtest: [basics, bootstraptoken, cacerts, compat, lazypull, upgrade]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 1
-    - name: "Download k3s image"
-      uses: actions/download-artifact@v4
-      with:
-        name: k3s
-        path: ./dist/artifacts
-    - name: Load k3s image
-      run: docker image load -i ./dist/artifacts/k3s-image.tar
-    - name: Run ${{ matrix.dtest }} Test
-      run: | 
-        chmod +x ./dist/artifacts/k3s
-        . ./scripts/version.sh
-        . ./tests/docker/test-helpers
-        . ./tests/docker/test-run-${{ matrix.dtest }}
-        echo "Did test-run-${{ matrix.dtest }} pass $?"
 
   build-go-tests:
     name: "Build Go Tests"
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     outputs:
-      branch_name: ${{ steps.branch_step.outputs.BRANCH_NAME }}
+      channel: ${{ steps.channel_step.outputs.channel }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -146,73 +126,66 @@ jobs:
     - name: Upload Go Tests
       uses: actions/upload-artifact@v4
       with:
-        name: docker-go-tests
+        name: docker-go-tests-${{ matrix.arch }}
         path: ./dist/artifacts/*.test
         compression-level: 9
         retention-days: 1
-    # For upgrade and skew tests, we need to know the branch name this run is based off.
-    # Since this is predetermined, we can run this step before the docker-go job, saving time.
-    # For PRs we can use the base_ref (ie the target branch of the PR).
-    # For pushes to k3s-io/k3s from dependabot or updatecli, use master
-    # All other pushes should be a valid ref, master or release-1.XX.
-    # For pushes to a fork, we need to determine the branch name by finding the parent branch from git show-branch history.
-    - name: Determine branch name
-      id: branch_step
+    # For upgrade and skew tests, we need to know the channel this run is based off.
+    # Since this is predetermined, we can run this step before the actual test job, saving time.
+    - name: Determine channel
+      id: channel_step
       run: |
-        if [ ${{ github.repository }} = "k3s-io/k3s" ]; then 
-          BRANCH_NAME=$(echo ${{ github.base_ref || github.ref_name }})
-          if [[ $BRANCH_NAME =~ ^(dependabot|updatecli) ]]; then
-            BRANCH_NAME=master
-          fi
-        elif [ -z "${{ github.base_ref }}" ]; then
-          # We are in a fork, and need some git history to determine the branch name
-          # For some reason, the first fetch doesn't always get the full history, so we sleep and fetch again
-          git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
-          sleep 5
-          git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
-          BRANCH_NAME=$(git show-branch -a 2> /dev/null  | grep '\*' | grep -v `git rev-parse --abbrev-ref HEAD` | head -n1 |  sed 's/.*\[\(.*\/\)\(.*\)\].*/\2/' | sed 's/[\^~].*//')
-        else
-          BRANCH_NAME=${{ github.base_ref }}
-        fi
-        echo "Branch Name is $BRANCH_NAME"
-        echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
-    # branch name should be either master or release-1.XX
-    - name: Fail if branch name does not match pattern
+        . ./scripts/version.sh
+        MINOR_VER=$(echo $VERSION_TAG | cut -d'.' -f1,2)
+        echo "CHANNEL=$MINOR_VER" >> $GITHUB_OUTPUT 
+    # channel name should be v1.XX or latest
+    - name: Fail if channel name does not match pattern
       run: |
-        if [[ ! ${{ steps.branch_step.outputs.branch_name }} =~ ^(master|release-[0-9]+\.[0-9]+)$ ]]; then
-          echo "Branch name ${{ steps.branch_step.outputs.branch_name }} does not match pattern"
-          echo "If this is a PR/fork, ensure you have recently rebased off master/release-1.XX branch"
+        if [[ ! ${{ steps.channel_step.outputs.channel }} =~ ^v1\.[0-9]+$|latest$ ]]; then
+          echo "Channel name ${{ steps.channel_step.outputs.channel }} does not match pattern"
           exit 1
         fi
-
+  
   docker-go:
-    needs: [build, build-go-tests]
-    name: Docker Tests In GO
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
+    needs: [build, build-arm64, build-go-tests]
+    name: Docker
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, upgrade]
+        dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, snapshotrestore, upgrade]
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     env:
-      BRANCH_NAME: ${{ needs.build-go-tests.outputs.branch_name }}
+      CHANNEL: ${{ needs.build-go-tests.outputs.channel }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - name: "Download K3s image"
+    - name: "Download K3s image (amd64)"
+      if: ${{ matrix.arch == 'amd64' }}
       uses: actions/download-artifact@v4
       with:
         name: k3s
         path: ./dist/artifacts
+    - name: "Download K3s image (arm64)"
+      if: ${{ matrix.arch == 'arm64' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: k3s-arm64
+        path: ./dist/artifacts
     - name: Load and set K3s image
       run: |
+        if [ ${{ matrix.arch }} = "arm64" ]; then
+          mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s
+        fi
+        chmod +x ./dist/artifacts/k3s
         docker image load -i ./dist/artifacts/k3s-image.tar
         IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s')
         echo "K3S_IMAGE=$IMAGE_TAG" >> $GITHUB_ENV
     - name: Download Go Tests
       uses: actions/download-artifact@v4
       with:
-        name: docker-go-tests
+        name: docker-go-tests-${{ matrix.arch }}
         path: ./dist/artifacts
     - name: Run ${{ matrix.dtest }} Test
       # Put the compiled test binary back in the same place as the test source
@@ -221,7 +194,9 @@ jobs:
         mv ./dist/artifacts/${{ matrix.dtest }}.test ./tests/docker/${{ matrix.dtest }}/
         cd ./tests/docker/${{ matrix.dtest }}
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
-          ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -branch=$BRANCH_NAME
+          ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
+        elif [ ${{ matrix.dtest }} = "snapshotrestore" ]; then
+          ./${{ matrix.dtest }}.test -ci 
         else
           ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE
         fi

Dockerfile.test

@@ -36,15 +36,23 @@ ENTRYPOINT ["./scripts/entry.sh"]
 CMD ["test"]
 
 
-FROM vagrantlibvirt/vagrant-libvirt:0.12.1 AS test-e2e
-RUN apt-get update && apt-get install -y docker.io
+FROM vagrantlibvirt/vagrant-libvirt:sha-a94ce0d AS test-e2e
+RUN apt-get update && apt-get install -y docker.io wget
+
 ENV VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1
 RUN vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
-RUN vagrant box add bento/ubuntu-24.04 --provider libvirt --force
+
+# Workaround for older vagrant-libvirt image and new vagrant infra wesbites
+# See https://github.com/hashicorp/vagrant/issues/13571 and
+# https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1840
+RUN wget https://app.vagrantup.com/bento/boxes/ubuntu-24.04/versions/202404.26.0/providers/libvirt.box -O bento-ubuntu24.04-202404.26.0.box
+RUN vagrant box add bento/ubuntu-24.04 bento-ubuntu24.04-202404.26.0.box
+RUN cd /.vagrant.d/boxes/bento-VAGRANTSLASH-ubuntu-24.04/ && mv 0 202404.26.0 && echo -n "https://app.vagrantup.com/bento/boxes/ubuntu-24.04" > metadata_url
+
 RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
     chmod +x ./kubectl; \
     mv ./kubectl /usr/local/bin/kubectl
-RUN GO_VERSION=go1.21.5; \
+RUN GO_VERSION=go1.22.5; \
     curl -O -L "https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz"; \
     rm -rf /usr/local/go; \
     tar -C /usr/local -xzf ${GO_VERSION}.linux-amd64.tar.gz;

scripts/test

@@ -19,41 +19,41 @@ mkdir -p $artifacts
 
 docker ps
 
+export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
+
 # ---
-# Only run basic tests on non amd64 archs, we use GitHub Actions for amd64
-if [ "$ARCH" != 'amd64' ]; then
+# Only run PR tests on arm arch, we use GitHub Actions for amd64 and arm64
+# Run all tests on tag events, as we want test failures to block the release
+if [ "$ARCH" == 'arm' ] ||  [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
 
-  export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
   go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE"
   echo "Did go test basics $?"
 
-  . ./tests/docker/test-run-basics
-  echo "Did test-run-basics $?"
+  # Extract v1.XX minor version for skew and upgrade tests
+  minor_version=$(echo $VERSION_K8S | cut -d '.' -f1,2)
 
-  . ./tests/docker/test-run-cacerts
-  echo "Did test-run-cacerts $?"
+  go test ./tests/docker/cacerts/cacerts_test.go -k3sImage="$K3S_IMAGE"
+  echo "Did go test cacerts $?"
 
-  . ./tests/docker/test-run-compat
-  echo "Did test-run-compat $?"
+  go test ./tests/docker/skew/skew_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version"
+  echo "Did go test skew $?"
 
-  . ./tests/docker/test-run-bootstraptoken
-  echo "Did test-run-bootstraptoken $?"
+  go test ./tests/docker/bootstraptoken/bootstraptoken_test.go -k3sImage="$K3S_IMAGE"
+  echo "Did go test bootstraptoken $?"
 
-  . ./tests/docker/test-run-upgrade
-  echo "Did test-run-upgrade $?"
+  go test ./tests/docker/upgrade/upgrade_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version"
+  echo "Did go test upgrade $?"
+
+  go test ./tests/docker/lazypull/lazypull_test.go -k3sImage="$K3S_IMAGE"
+  echo "Did go test lazypull $?"
 
-  . ./tests/docker/test-run-lazypull
-  echo "Did test-run-lazypull $?"
 fi
 
 
-
+#TODO convert this to new go test framework
 . ./tests/docker/test-run-hardened
 echo "Did test-run-hardened $?"
 
-. ./tests/docker/test-run-etcd
-echo "Did test-run-etcd $?"
-
 # ---
 
 [ "$ARCH" != 'amd64' ] && \
@@ -71,10 +71,10 @@ fi
 # ---
 
 if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
-  E2E_OUTPUT=$artifacts test-run-sonobuoy serial
-  echo "Did test-run-sonobuoy serial $?"
-  test-run-sonobuoy etcd serial
-  echo "Did test-run-sonobuoy-etcd serial $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -serial -ginkgo.v
+  echo "Did go conformance sqlite serial $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -serial -ginkgo.v
+  echo "Did go conformance etcd serial $?"
   test-run-sonobuoy mysql serial
   echo "Did test-run-sonobuoy-mysqk serial $?"
   test-run-sonobuoy postgres serial
@@ -91,8 +91,10 @@ if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
 
   E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
   echo "Did test-run-sonobuoy parallel $?"
-  test-run-sonobuoy etcd parallel
-  echo "Did test-run-sonobuoy-etcd parallel $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -ginkgo.v
+  echo "Did go conformance sqlite parallel $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -ginkgo.v
+  echo "Did go test conformance etcd parallel $?"
   test-run-sonobuoy mysql parallel
   echo "Did test-run-sonobuoy-mysql parallel $?"
   test-run-sonobuoy postgres parallel