diff --git a/config/default/manager/manager.yaml b/config/default/manager/manager.yaml index 33bb6f882..4d9021a0c 100644 --- a/config/default/manager/manager.yaml +++ b/config/default/manager/manager.yaml @@ -5,6 +5,8 @@ metadata: control-plane: mac-controller-manager controller-tools.k8s.io: "1.0" kubemacpool/ignoreAdmission: "true" + runlevel: "0" + openshift.io/run-level: "0" name: system --- apiVersion: v1 diff --git a/config/release/kubemacpool.yaml b/config/release/kubemacpool.yaml index 5218b6eb1..840d62174 100644 --- a/config/release/kubemacpool.yaml +++ b/config/release/kubemacpool.yaml @@ -5,6 +5,8 @@ metadata: control-plane: mac-controller-manager controller-tools.k8s.io: "1.0" kubemacpool/ignoreAdmission: "true" + openshift.io/run-level: "0" + runlevel: "0" name: kubemacpool-system --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/config/test/kubemacpool.yaml b/config/test/kubemacpool.yaml index bab5c8950..8446051a0 100644 --- a/config/test/kubemacpool.yaml +++ b/config/test/kubemacpool.yaml @@ -5,6 +5,8 @@ metadata: control-plane: mac-controller-manager controller-tools.k8s.io: "1.0" kubemacpool/ignoreAdmission: "true" + openshift.io/run-level: "0" + runlevel: "0" name: kubemacpool-system --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/pkg/names/names.go b/pkg/names/names.go index a26e9172e..8c79544a4 100644 --- a/pkg/names/names.go +++ b/pkg/names/names.go @@ -15,3 +15,9 @@ const LEADER_LABEL = "kubemacpool-leader" const LEADER_ID = "kubemacpool-election" const ADMISSION_IGNORE_LABEL = "kubemacpool/ignoreAdmission" + +const K8S_RUNLABEL = "runlevel" + +const OPENSHIFT_RUNLABEL = "openshift.io/run-level" + +var CRITICAL_RUNLABELS = []string{"0", "1"} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 1f159d1fc..b04ab4b20 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -67,8 +67,14 @@ func AddToManager(mgr manager.Manager, poolManager *pool_manager.PoolManager, ma return err } - namespaceSelector := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: names.ADMISSION_IGNORE_LABEL, - Operator: metav1.LabelSelectorOpDoesNotExist}}} + namespaceSelector := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: names.ADMISSION_IGNORE_LABEL, + Operator: metav1.LabelSelectorOpDoesNotExist}, + {Key: names.K8S_RUNLABEL, + Operator: metav1.LabelSelectorOpNotIn, Values: names.CRITICAL_RUNLABELS}, + {Key: names.OPENSHIFT_RUNLABEL, + Operator: metav1.LabelSelectorOpNotIn, Values: names.CRITICAL_RUNLABELS, + }}} webhooks := []runtimewebhook.Webhook{} for _, f := range AddToManagerFuncs { diff --git a/tests/pods_test.go b/tests/pods_test.go index 10bb15cf1..2d4c1667c 100644 --- a/tests/pods_test.go +++ b/tests/pods_test.go @@ -1,16 +1,98 @@ package tests import ( + "context" + "fmt" + "strings" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/onsi/gomega/types" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/k8snetworkplumbingwg/kubemacpool/pkg/names" ) +const defaultNumberOfReplicas = 2 + var _ = Describe("Pods", func() { - Context("Check the client", func() { - It("should not fail", func() { - _, err := testClient.KubeClient.CoreV1().Pods("").List(v1.ListOptions{}) + Context("Check the pod mutating webhook", func() { + AfterEach(func() { + // Clean pods from our test namespaces after every test to start clean + for _, namespace := range []string{TestNamespace, OtherTestNamespace} { + podList := &corev1.PodList{} + err := testClient.VirtClient.List(context.TODO(), &client.ListOptions{Namespace: namespace}, podList) + Expect(err).ToNot(HaveOccurred()) + + for _, podObject := range podList.Items { + err = testClient.VirtClient.Delete(context.TODO(), &podObject) + Expect(err).ToNot(HaveOccurred()) + } + + Eventually(func() int { + podList := &corev1.PodList{} + err := testClient.VirtClient.List(context.TODO(), &client.ListOptions{Namespace: namespace}, podList) + Expect(err).ToNot(HaveOccurred()) + return len(podList.Items) + + }, timeout, pollingInterval).Should(Equal(0), fmt.Sprintf("failed to remove all pod objects from namespace %s", namespace)) + + // This function remove all the labels from the namespace + err = cleanNamespaceLabels(namespace) + } + + // Restore the default number of managers + err := changeManagerReplicas(defaultNumberOfReplicas) Expect(err).ToNot(HaveOccurred()) }) + + testCriticalNamespace := func(namespace, label string, matcher types.GomegaMatcher) { + err := changeManagerReplicas(0) + Expect(err).ToNot(HaveOccurred()) + + err = addLabelsToNamespace(OtherTestNamespace, map[string]string{label: "0"}) + + podObject := createPodObject() + + Eventually(func() bool { + _, err := testClient.KubeClient.CoreV1().Pods(OtherTestNamespace).Create(podObject) + if err != nil && strings.Contains(err.Error(), "connection refused") { + return false + } + + return true + }, timeout, pollingInterval).Should(matcher, "failed to apply the new pod object") + } + + It("should create a pod when mac pool is running in a regular namespace", func() { + err := setRange(rangeStart, rangeEnd) + Expect(err).ToNot(HaveOccurred()) + + podObject := createPodObject() + + Eventually(func() bool { + _, err := testClient.KubeClient.CoreV1().Pods(TestNamespace).Create(podObject) + if err != nil && strings.Contains(err.Error(), "connection refused") { + return false + } + + return true + }, timeout, pollingInterval).Should(BeTrue(), "failed to apply the new pod object") + }) + + // We never fail thanks to the "Ignore" failure policy + It("should create a pod on a regular namespace when mac pool is down", func() { + testCriticalNamespace(OtherTestNamespace, "not-critical", BeTrue()) + }) + + It("should create a pod on a critical k8s namespaces when mac pool is down", func() { + testCriticalNamespace(OtherTestNamespace, names.K8S_RUNLABEL, BeTrue()) + }) + + It("should create a pod on a critical openshift namespaces when mac pool is down", func() { + testCriticalNamespace(OtherTestNamespace, names.OPENSHIFT_RUNLABEL, BeTrue()) + }) }) }) diff --git a/tests/tests.go b/tests/tests.go index c37bfab8e..18733353e 100644 --- a/tests/tests.go +++ b/tests/tests.go @@ -33,7 +33,7 @@ const ( ) var ( - gracePeriodSeconds int64 = 10 + gracePeriodSeconds int64 = 3 rangeStart = "02:00:00:00:00:00" rangeEnd = "02:FF:FF:FF:FF:FF" testClient *TestClient @@ -109,6 +109,23 @@ func CreateVmObject(namespace string, running bool, interfaces []kubevirtv1.Inte return vm } +func createPodObject() *corev1.Pod { + podName := "testpod" + rand.String(32) + podObject := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podName}, + Spec: corev1.PodSpec{TerminationGracePeriodSeconds: &gracePeriodSeconds, + Containers: []corev1.Container{{Name: "test", + Image: "centos", + Command: []string{"/bin/bash", "-c", "sleep INF"}}}}} + + return &podObject +} + +func addNetworksToPod(pod *corev1.Pod, networks []map[string]string) { + if networks != nil && len(networks) > 0 { + pod.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%v", networks)} + } +} + func setRange(rangeStart, rangeEnd string) error { configMap, err := testClient.KubeClient.CoreV1().ConfigMaps(ManagerNamespce).Get("kubemacpool-mac-range-config", metav1.GetOptions{}) if err != nil { @@ -135,14 +152,19 @@ func setRange(rangeStart, rangeEnd string) error { } } + macDeploy, err := testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Get(names.MANAGER_DEPLOYMENT, metav1.GetOptions{}) + if err != nil { + return err + } + Eventually(func() error { podsList, err = testClient.KubeClient.CoreV1().Pods(ManagerNamespce).List(metav1.ListOptions{}) if err != nil { return err } - if len(podsList.Items) != 2 { - return fmt.Errorf("should have two manager pods") + if len(podsList.Items) != int(macDeploy.Status.Replicas) { + return fmt.Errorf("should have %v manager pods", macDeploy.Status.Replicas) } for _, pod := range podsList.Items { @@ -194,6 +216,89 @@ func DeleteLeaderManager() { time.Sleep(40 * time.Second) } +func changeManagerReplicas(numOfReplica int32) error { + Eventually(func() error { + managerDeployment, err := testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Get(names.MANAGER_DEPLOYMENT, metav1.GetOptions{}) + if err != nil { + return err + } + + managerDeployment.Spec.Replicas = &numOfReplica + + _, err = testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Update(managerDeployment) + if err != nil { + return err + } + + return nil + }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred(), "failed to update number of replicas on manager") + + Eventually(func() bool { + managerDeployment, err := testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Get(names.MANAGER_DEPLOYMENT, metav1.GetOptions{}) + if err != nil { + return false + } + + if managerDeployment.Status.Replicas != numOfReplica { + return false + } + + if managerDeployment.Status.ReadyReplicas != numOfReplica { + return false + } + + podsList, err := testClient.KubeClient.CoreV1().Pods(ManagerNamespce).List(metav1.ListOptions{}) + if err != nil { + return false + } + + if len(podsList.Items) != int(numOfReplica) { + return false + } + + for _, podObject := range podsList.Items { + if podObject.Status.Phase != corev1.PodRunning { + return false + } + } + + return true + + }, 30*time.Second, 3*time.Second).Should(BeTrue(), "failed to change kubemacpool deployment number of replicas") + + return nil +} + +func cleanNamespaceLabels(namespace string) error { + nsObject, err := testClient.KubeClient.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}) + if err != nil { + return err + } + + nsObject.Labels = make(map[string]string) + + _, err = testClient.KubeClient.CoreV1().Namespaces().Update(nsObject) + return err +} + +func addLabelsToNamespace(namespace string, labels map[string]string) error { + nsObject, err := testClient.KubeClient.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}) + if err != nil { + return err + } + + if nsObject.Labels == nil { + nsObject.Labels = labels + } else { + for key, value := range labels { + nsObject.Labels[key] = value + } + } + + _, err = testClient.KubeClient.CoreV1().Namespaces().Update(nsObject) + return err +} + func BeforeAll(fn func()) { first := true BeforeEach(func() {