Merge pull request #107 from phoracek/backport_ignoring_of_ns

[release-v0.8] Skip pods in critical namespaces

config/default/manager/manager.yaml

@@ -5,6 +5,8 @@ metadata:
     control-plane: mac-controller-manager
     controller-tools.k8s.io: "1.0"
     kubemacpool/ignoreAdmission: "true"
+    runlevel: "0"
+    openshift.io/run-level: "0"
   name: system
 ---
 apiVersion: v1

config/release/kubemacpool.yaml

@@ -5,6 +5,8 @@ metadata:
     control-plane: mac-controller-manager
     controller-tools.k8s.io: "1.0"
     kubemacpool/ignoreAdmission: "true"
+    openshift.io/run-level: "0"
+    runlevel: "0"
   name: kubemacpool-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1

config/test/kubemacpool.yaml

@@ -5,6 +5,8 @@ metadata:
     control-plane: mac-controller-manager
     controller-tools.k8s.io: "1.0"
     kubemacpool/ignoreAdmission: "true"
+    openshift.io/run-level: "0"
+    runlevel: "0"
   name: kubemacpool-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1

pkg/names/names.go

@@ -15,3 +15,9 @@ const LEADER_LABEL = "kubemacpool-leader"
 const LEADER_ID = "kubemacpool-election"
 
 const ADMISSION_IGNORE_LABEL = "kubemacpool/ignoreAdmission"
+
+const K8S_RUNLABEL = "runlevel"
+
+const OPENSHIFT_RUNLABEL = "openshift.io/run-level"
+
+var CRITICAL_RUNLABELS = []string{"0", "1"}

pkg/webhook/webhook.go

@@ -67,8 +67,14 @@ func AddToManager(mgr manager.Manager, poolManager *pool_manager.PoolManager, ma
 		return err
 	}
 
-	namespaceSelector := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: names.ADMISSION_IGNORE_LABEL,
-		Operator: metav1.LabelSelectorOpDoesNotExist}}}
+	namespaceSelector := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+		{Key: names.ADMISSION_IGNORE_LABEL,
+			Operator: metav1.LabelSelectorOpDoesNotExist},
+		{Key: names.K8S_RUNLABEL,
+			Operator: metav1.LabelSelectorOpNotIn, Values: names.CRITICAL_RUNLABELS},
+		{Key: names.OPENSHIFT_RUNLABEL,
+			Operator: metav1.LabelSelectorOpNotIn, Values: names.CRITICAL_RUNLABELS,
+		}}}
 
 	webhooks := []runtimewebhook.Webhook{}
 	for _, f := range AddToManagerFuncs {

tests/pods_test.go

@@ -1,16 +1,98 @@
 package tests
 
 import (
+	"context"
+	"fmt"
+	"strings"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"github.com/onsi/gomega/types"
+
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/k8snetworkplumbingwg/kubemacpool/pkg/names"
 )
 
+const defaultNumberOfReplicas = 2
+
 var _ = Describe("Pods", func() {
-	Context("Check the client", func() {
-		It("should not fail", func() {
-			_, err := testClient.KubeClient.CoreV1().Pods("").List(v1.ListOptions{})
+	Context("Check the pod mutating webhook", func() {
+		AfterEach(func() {
+			// Clean pods from our test namespaces after every test to start clean
+			for _, namespace := range []string{TestNamespace, OtherTestNamespace} {
+				podList := &corev1.PodList{}
+				err := testClient.VirtClient.List(context.TODO(), &client.ListOptions{Namespace: namespace}, podList)
+				Expect(err).ToNot(HaveOccurred())
+
+				for _, podObject := range podList.Items {
+					err = testClient.VirtClient.Delete(context.TODO(), &podObject)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				Eventually(func() int {
+					podList := &corev1.PodList{}
+					err := testClient.VirtClient.List(context.TODO(), &client.ListOptions{Namespace: namespace}, podList)
+					Expect(err).ToNot(HaveOccurred())
+					return len(podList.Items)
+
+				}, timeout, pollingInterval).Should(Equal(0), fmt.Sprintf("failed to remove all pod objects from namespace %s", namespace))
+
+				// This function remove all the labels from the namespace
+				err = cleanNamespaceLabels(namespace)
+			}
+
+			// Restore the default number of managers
+			err := changeManagerReplicas(defaultNumberOfReplicas)
 			Expect(err).ToNot(HaveOccurred())
 		})
+
+		testCriticalNamespace := func(namespace, label string, matcher types.GomegaMatcher) {
+			err := changeManagerReplicas(0)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = addLabelsToNamespace(OtherTestNamespace, map[string]string{label: "0"})
+
+			podObject := createPodObject()
+
+			Eventually(func() bool {
+				_, err := testClient.KubeClient.CoreV1().Pods(OtherTestNamespace).Create(podObject)
+				if err != nil && strings.Contains(err.Error(), "connection refused") {
+					return false
+				}
+
+				return true
+			}, timeout, pollingInterval).Should(matcher, "failed to apply the new pod object")
+		}
+
+		It("should create a pod when mac pool is running in a regular namespace", func() {
+			err := setRange(rangeStart, rangeEnd)
+			Expect(err).ToNot(HaveOccurred())
+
+			podObject := createPodObject()
+
+			Eventually(func() bool {
+				_, err := testClient.KubeClient.CoreV1().Pods(TestNamespace).Create(podObject)
+				if err != nil && strings.Contains(err.Error(), "connection refused") {
+					return false
+				}
+
+				return true
+			}, timeout, pollingInterval).Should(BeTrue(), "failed to apply the new pod object")
+		})
+
+		// We never fail thanks to the "Ignore" failure policy
+		It("should create a pod on a regular namespace when mac pool is down", func() {
+			testCriticalNamespace(OtherTestNamespace, "not-critical", BeTrue())
+		})
+
+		It("should create a pod on a critical k8s namespaces when mac pool is down", func() {
+			testCriticalNamespace(OtherTestNamespace, names.K8S_RUNLABEL, BeTrue())
+		})
+
+		It("should create a pod on a critical openshift namespaces when mac pool is down", func() {
+			testCriticalNamespace(OtherTestNamespace, names.OPENSHIFT_RUNLABEL, BeTrue())
+		})
 	})
 })

tests/tests.go

@@ -33,7 +33,7 @@ const (
 )
 
 var (
-	gracePeriodSeconds int64 = 10
+	gracePeriodSeconds int64 = 3
 	rangeStart               = "02:00:00:00:00:00"
 	rangeEnd                 = "02:FF:FF:FF:FF:FF"
 	testClient         *TestClient
@@ -109,6 +109,23 @@ func CreateVmObject(namespace string, running bool, interfaces []kubevirtv1.Inte
 	return vm
 }
 
+func createPodObject() *corev1.Pod {
+	podName := "testpod" + rand.String(32)
+	podObject := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podName},
+		Spec: corev1.PodSpec{TerminationGracePeriodSeconds: &gracePeriodSeconds,
+			Containers: []corev1.Container{{Name: "test",
+				Image:   "centos",
+				Command: []string{"/bin/bash", "-c", "sleep INF"}}}}}
+
+	return &podObject
+}
+
+func addNetworksToPod(pod *corev1.Pod, networks []map[string]string) {
+	if networks != nil && len(networks) > 0 {
+		pod.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%v", networks)}
+	}
+}
+
 func setRange(rangeStart, rangeEnd string) error {
 	configMap, err := testClient.KubeClient.CoreV1().ConfigMaps(ManagerNamespce).Get("kubemacpool-mac-range-config", metav1.GetOptions{})
 	if err != nil {
@@ -135,14 +152,19 @@ func setRange(rangeStart, rangeEnd string) error {
 		}
 	}
 
+	macDeploy, err := testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Get(names.MANAGER_DEPLOYMENT, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
 	Eventually(func() error {
 		podsList, err = testClient.KubeClient.CoreV1().Pods(ManagerNamespce).List(metav1.ListOptions{})
 		if err != nil {
 			return err
 		}
 
-		if len(podsList.Items) != 2 {
-			return fmt.Errorf("should have two manager pods")
+		if len(podsList.Items) != int(macDeploy.Status.Replicas) {
+			return fmt.Errorf("should have %v manager pods", macDeploy.Status.Replicas)
 		}
 
 		for _, pod := range podsList.Items {
@@ -194,6 +216,89 @@ func DeleteLeaderManager() {
 	time.Sleep(40 * time.Second)
 }
 
+func changeManagerReplicas(numOfReplica int32) error {
+	Eventually(func() error {
+		managerDeployment, err := testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Get(names.MANAGER_DEPLOYMENT, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		managerDeployment.Spec.Replicas = &numOfReplica
+
+		_, err = testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Update(managerDeployment)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred(), "failed to update number of replicas on manager")
+
+	Eventually(func() bool {
+		managerDeployment, err := testClient.KubeClient.AppsV1().Deployments(ManagerNamespce).Get(names.MANAGER_DEPLOYMENT, metav1.GetOptions{})
+		if err != nil {
+			return false
+		}
+
+		if managerDeployment.Status.Replicas != numOfReplica {
+			return false
+		}
+
+		if managerDeployment.Status.ReadyReplicas != numOfReplica {
+			return false
+		}
+
+		podsList, err := testClient.KubeClient.CoreV1().Pods(ManagerNamespce).List(metav1.ListOptions{})
+		if err != nil {
+			return false
+		}
+
+		if len(podsList.Items) != int(numOfReplica) {
+			return false
+		}
+
+		for _, podObject := range podsList.Items {
+			if podObject.Status.Phase != corev1.PodRunning {
+				return false
+			}
+		}
+
+		return true
+
+	}, 30*time.Second, 3*time.Second).Should(BeTrue(), "failed to change kubemacpool deployment number of replicas")
+
+	return nil
+}
+
+func cleanNamespaceLabels(namespace string) error {
+	nsObject, err := testClient.KubeClient.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	nsObject.Labels = make(map[string]string)
+
+	_, err = testClient.KubeClient.CoreV1().Namespaces().Update(nsObject)
+	return err
+}
+
+func addLabelsToNamespace(namespace string, labels map[string]string) error {
+	nsObject, err := testClient.KubeClient.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if nsObject.Labels == nil {
+		nsObject.Labels = labels
+	} else {
+		for key, value := range labels {
+			nsObject.Labels[key] = value
+		}
+	}
+
+	_, err = testClient.KubeClient.CoreV1().Namespaces().Update(nsObject)
+	return err
+}
+
 func BeforeAll(fn func()) {
 	first := true
 	BeforeEach(func() {