Trying wiht a job instead

Signed-off-by: Manuel Buil <[email protected]>

docs/adrs/multus.md

@@ -38,6 +38,10 @@ It sucks a bit that the daemonset stays dormant forever after doing the job inst
 
 * K3s includes the multus and whereabouts CNI plugins as part of its multi-exec cni binary. However, the whereabouts binary is using very old dependencies which would creep in CVEs. Moreover, the size of the K3s binary would increase more than 10%, something not acceptable for a something that the vast majority of K3s users will not enable
 
+* Use a helm chart where we specify the datadir where it should deploy the extra plugin binaries (e.g. multus), we call that directory the CNI bin directory. The problem is that all nodes in the cluster do not necessarily share the same CNI bin directory and if we use /var/lib/rancher/data/$SHA/bin, that $SHA will change with each K3s build
+
+
+
 
 ### Limitations
 

manifests/rolebindings.yaml

@@ -47,6 +47,9 @@ rules:
   - list
   - get
   - watch
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["create"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

pkg/agent/config/config.go

@@ -508,6 +508,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
 		FlannelExternalIP:        controlConfig.FlannelExternalIP,
+		Multus:                   controlConfig.Multus,
 		EgressSelectorMode:       controlConfig.EgressSelectorMode,
 		ServerHTTPSPort:          controlConfig.HTTPSPort,
 		Token:                    info.String(),

pkg/agent/multus/multus.go

@@ -0,0 +1,128 @@
+package multus
+
+// Generate a code which creates a kubernetes Job object with an image
+// and uses a selector to only deploy in a node, whose name can be be picked up from the config AgentConfig.NodeName
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func StartMultusJob(ctx context.Context, nodeConfig *config.Node) error {
+	imageName, err := fetchImageName(ctx, nodeConfig)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("MANU - This is the imageName: %v\n", imageName)
+
+	job := generateJob(nodeConfig.AgentConfig.NodeName, imageName, 	nodeConfig.AgentConfig.CNIBinDir)
+
+	fmt.Printf("MANU - This is the job: %v\n", job)
+	config, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)
+	if err != nil {
+		return err
+	}
+
+	// Create a new clientset
+	clientset, _ := kubernetes.NewForConfig(config)
+	_, err = clientset.BatchV1().Jobs("kube-system").Create(context.TODO(), &job, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// fetchImageName() is a function that returns the image name by using helm and reading the initContainer of the multus chart
+func fetchImageName(ctx context.Context, nodeConfig *config.Node) (string, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)
+	if err != nil {
+			return "", err
+	}
+
+	// Create a new clientset
+    clientset, _ := kubernetes.NewForConfig(config)
+    daemonset, _ := clientset.AppsV1().DaemonSets("kube-system").Get(context.TODO(), "multus", metav1.GetOptions{})
+
+	var imageName string
+	if err := wait.PollImmediateWithContext(ctx, 5*time.Second, 100*time.Second, func(ctx context.Context) (bool, error) {
+	    for _, initContainer := range daemonset.Spec.Template.Spec.InitContainers {
+    	    if initContainer.Name == "cni-plugins" {
+        	    imageName = initContainer.Image
+				fmt.Printf("MANU - ImageName found! %v\n", imageName)
+				return true, nil
+        	}
+    	}
+		fmt.Printf("MANU - Could not find the multus initContainer image, trying again in 10 seconds\n")
+		return false, nil
+	}); err != nil {
+		return "", fmt.Errorf("time out trying to find the multus initContainer image")
+	}
+
+	return imageName, nil
+}
+
+func generateJob(nodeName, imageName, cniBinDir string) batchv1.Job {
+	hostPathDirectoryOrCreate := corev1.HostPathDirectoryOrCreate
+	// Create a new Job object
+	job := batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "multus",
+		},
+		Spec: batchv1.JobSpec{
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "multus",
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "multus",
+							Image: imageName,
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									MountPath: "/host/opt/cni/bin",
+									Name:      "cni-path",
+								},
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name:  "SKIP_CNI_BINARIES",
+									Value: "flannel",
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: "cni-path",
+							VolumeSource: corev1.VolumeSource{
+								HostPath: &corev1.HostPathVolumeSource{
+									Path: cniBinDir,
+									Type: &hostPathDirectoryOrCreate,
+								},
+							},
+						},
+					},
+					NodeSelector: map[string]string{
+						"kubernetes.io/hostname": nodeName,
+					},
+				},
+			},
+		},
+	}
+
+	return job
+}

pkg/agent/run.go

@@ -15,6 +15,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/config"
 	"github.com/k3s-io/k3s/pkg/agent/containerd"
 	"github.com/k3s-io/k3s/pkg/agent/flannel"
+	"github.com/k3s-io/k3s/pkg/agent/multus"
 	"github.com/k3s-io/k3s/pkg/agent/netpol"
 	"github.com/k3s-io/k3s/pkg/agent/proxy"
 	"github.com/k3s-io/k3s/pkg/agent/syssetup"
@@ -129,6 +130,12 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		if err := flannel.Prepare(ctx, nodeConfig); err != nil {
 			return err
 		}
+
+		if nodeConfig.Multus {
+			if err := multus.StartMultusJob(ctx, nodeConfig); err != nil {
+				return err
+			}
+		}
 	}
 
 	if nodeConfig.Docker {

pkg/daemons/config/types.go

@@ -40,6 +40,7 @@ type Node struct {
 	ContainerRuntimeEndpoint string
 	ImageServiceEndpoint     string
 	NoFlannel                bool
+	Multus                   bool
 	SELinux                  bool
 	EmbeddedRegistry         bool
 	FlannelBackend           string
@@ -157,7 +158,7 @@ type CriticalControlArgs struct {
 	FlannelIPv6Masq       bool         `cli:"flannel-ipv6-masq"`
 	FlannelExternalIP     bool         `cli:"flannel-external-ip"`
 	EgressSelectorMode    string       `cli:"egress-selector-mode"`
-	Multus				  bool         `cli:"multus"`	
+	Multus                bool         `cli:"multus"`
 	ServiceIPRange        *net.IPNet   `cli:"service-cidr"`
 	ServiceIPRanges       []*net.IPNet `cli:"service-cidr"`
 }