e2e: Add PersistentIPs tests

Signed-off-by: Enrique Llorente <[email protected]>

Makefile

@@ -67,7 +67,10 @@ test: manifests generate fmt vet envtest ## Run tests.
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
-	go test ./test/e2e/ -v -ginkgo.v
+	export KUBECONFIG=$$(pwd)/.output/kubeconfig && \
+	export PATH=$$(pwd)/.output/ovn-kubernetes/bin:$${PATH} && \
+	cd test/e2e && \
+	go test -v --ginkgo.v
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint

test/e2e/e2e_suite_test.go

@@ -21,8 +21,19 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	testenv "github.com/maiqueb/kubevirt-ipam-claims/test/env"
 )
 
+var _ = BeforeSuite(func() {
+	ctrl.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	testenv.Start()
+})
+
 // Run e2e tests using the Ginkgo runner.
 func TestE2E(t *testing.T) {
 	RegisterFailHandler(Fail)

test/e2e/persistentips_test.go

@@ -0,0 +1,175 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2024 Red Hat, Inc.
+ *
+ */
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+
+	kubevirtv1 "kubevirt.io/api/core/v1"
+
+	testenv "github.com/maiqueb/kubevirt-ipam-claims/test/env"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("Persistent IPs", func() {
+	When("network attachment definition created with allowPersistentIPs=true", func() {
+		var (
+			td                   testenv.TestData
+			networkInterfaceName = "multus"
+			vm                   *kubevirtv1.VirtualMachine
+			vmi                  *kubevirtv1.VirtualMachineInstance
+			nad                  *nadv1.NetworkAttachmentDefinition
+		)
+		BeforeEach(func() {
+			td = testenv.GenerateTestData()
+			td.SetUp()
+			DeferCleanup(func() {
+				td.TearDown()
+			})
+
+			nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace)
+			vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, networkInterfaceName, nad.Name)
+			vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning())
+
+			By("Create NetworkAttachmentDefinition")
+			Expect(testenv.Client.Create(context.Background(), nad)).To(Succeed())
+		})
+		Context("and a virtual machine using it is also created", func() {
+			BeforeEach(func() {
+				By("Creating VM using the nad")
+				Expect(testenv.Client.Create(context.Background(), vm)).To(Succeed())
+
+				By(fmt.Sprintf("Waiting for readiness at virtual machine %s", vm.Name))
+				Eventually(testenv.ThisVMReadiness(vm)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(BeTrue())
+
+				By("Wait for IPAMClaim to get created")
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					ShouldNot(BeEmpty())
+
+				Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed())
+
+				Expect(vmi.Status.Interfaces).NotTo(BeEmpty())
+				Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty())
+			})
+
+			It("should keep ips after live migration", func() {
+				vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs
+
+				testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name)
+				testenv.CheckLiveMigrationSucceeded(td.Namespace, vm.Name)
+
+				By("Wait for VMI to be ready after live migration")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration)))
+
+			})
+
+			It("should garbage collect IPAMClaims after virtual machine deletion", func() {
+				Expect(testenv.Client.Delete(context.Background(), vm)).To(Succeed())
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeEmpty())
+			})
+
+			It("should keep ips after restart", func() {
+				vmiIPsBeforeRestart := vmi.Status.Interfaces[0].IPs
+				vmiUUIDBeforeRestart := vmi.UID
+
+				By("Re-starting the VM")
+				output, err := exec.Command("virtctl", "restart", "-n", td.Namespace, vmi.Name).CombinedOutput()
+				Expect(err).NotTo(HaveOccurred(), output)
+
+				By("Wait for a new VMI to be re-started")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(90 * time.Second).
+					Should(testenv.BeRestarted(vmiUUIDBeforeRestart))
+
+				By("Wait for VMI to be ready after restart")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeRestart)))
+			})
+		})
+		Context("and a virtual machine instance using it is also created", func() {
+			BeforeEach(func() {
+				By("Creating VMI using the nad")
+				Expect(testenv.Client.Create(context.Background(), vmi)).To(Succeed())
+
+				By(fmt.Sprintf("Waiting for readiness at virtual machine instance %s", vmi.Name))
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				By("Wait for IPAMClaim to get created")
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					ShouldNot(BeEmpty())
+
+				Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed())
+
+				Expect(vmi.Status.Interfaces).NotTo(BeEmpty())
+				Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty())
+			})
+
+			It("should keep ips after live migration", func() {
+				vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs
+
+				testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name)
+				testenv.CheckLiveMigrationSucceeded(td.Namespace, vmi.Name)
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration)))
+
+			})
+
+			It("should garbage collect IPAMClaims after virtual machine deletion", func() {
+				Expect(testenv.Client.Delete(context.Background(), vmi)).To(Succeed())
+				Eventually(testenv.IPAMClaimsFromNamespace(vmi.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeEmpty())
+			})
+		})
+
+	})
+})

test/env/compose.go

@@ -0,0 +1,97 @@
+package env
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+
+	kubevirtv1 "kubevirt.io/api/core/v1"
+)
+
+func ComposeLayer2WithSubnetNAD(namespace string) *nadv1.NetworkAttachmentDefinition {
+	networkName := "l2"
+	nadName := RandomName(networkName, 16)
+	return &nadv1.NetworkAttachmentDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      nadName,
+		},
+		Spec: nadv1.NetworkAttachmentDefinitionSpec{
+			Config: fmt.Sprintf(`
+{
+        "cniVersion": "0.3.0",
+        "name": "%[3]s",
+        "type": "ovn-k8s-cni-overlay",
+        "topology": "layer2",
+        "subnets": "10.100.200.0/24",
+        "netAttachDefName": "%[1]s/%[2]s",
+        "allowPersistentIPs": true
+}
+`, namespace, nadName, networkName),
+		},
+	}
+}
+
+func ComposeAlpineWithMultusVMI(namespace, interfaceName, networkName string) *kubevirtv1.VirtualMachineInstance {
+	return &kubevirtv1.VirtualMachineInstance{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      RandomName("alpine", 16),
+		},
+		Spec: kubevirtv1.VirtualMachineInstanceSpec{
+			Domain: kubevirtv1.DomainSpec{
+				Resources: kubevirtv1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceMemory: resource.MustParse("128Mi"),
+					},
+				},
+				Devices: kubevirtv1.Devices{
+					Disks: []kubevirtv1.Disk{
+						{
+							DiskDevice: kubevirtv1.DiskDevice{
+								Disk: &kubevirtv1.DiskTarget{
+									Bus: kubevirtv1.DiskBusVirtio,
+								},
+							},
+							Name: "containerdisk",
+						},
+					},
+					Interfaces: []kubevirtv1.Interface{
+						{
+							Name: interfaceName,
+							InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{
+								Bridge: &kubevirtv1.InterfaceBridge{},
+							},
+						},
+					},
+				},
+			},
+			Networks: []kubevirtv1.Network{
+				{
+					Name: interfaceName,
+					NetworkSource: kubevirtv1.NetworkSource{
+						Multus: &kubevirtv1.MultusNetwork{
+							NetworkName: networkName,
+						},
+					},
+				},
+			},
+			TerminationGracePeriodSeconds: pointer.Int64(5),
+			Volumes: []kubevirtv1.Volume{
+				{
+					Name: "containerdisk",
+					VolumeSource: kubevirtv1.VolumeSource{
+						ContainerDisk: &kubevirtv1.ContainerDiskSource{
+							Image: "quay.io/kubevirtci/alpine-container-disk-demo:devel_alt",
+						},
+					},
+				},
+			},
+		},
+	}
+}