diff --git a/docs/adrs/multus.md b/docs/adrs/multus.md index dc6dcee2f386..2f9f9f7ee37b 100644 --- a/docs/adrs/multus.md +++ b/docs/adrs/multus.md @@ -38,6 +38,10 @@ It sucks a bit that the daemonset stays dormant forever after doing the job inst * K3s includes the multus and whereabouts CNI plugins as part of its multi-exec cni binary. However, the whereabouts binary is using very old dependencies which would creep in CVEs. Moreover, the size of the K3s binary would increase more than 10%, something not acceptable for a something that the vast majority of K3s users will not enable +* Use a helm chart where we specify the datadir where it should deploy the extra plugin binaries (e.g. multus), we call that directory the CNI bin directory. The problem is that all nodes in the cluster do not necessarily share the same CNI bin directory and if we use /var/lib/rancher/data/$SHA/bin, that $SHA will change with each K3s build + + + ### Limitations diff --git a/manifests/rolebindings.yaml b/manifests/rolebindings.yaml index 9a4d1f655662..34c5001c8267 100644 --- a/manifests/rolebindings.yaml +++ b/manifests/rolebindings.yaml @@ -47,6 +47,9 @@ rules: - list - get - watch +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 4a16edd68a32..f9f75b1eb58d 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -508,6 +508,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N FlannelBackend: controlConfig.FlannelBackend, FlannelIPv6Masq: controlConfig.FlannelIPv6Masq, FlannelExternalIP: controlConfig.FlannelExternalIP, + Multus: controlConfig.Multus, EgressSelectorMode: controlConfig.EgressSelectorMode, ServerHTTPSPort: controlConfig.HTTPSPort, Token: info.String(), diff --git a/pkg/agent/multus/multus.go b/pkg/agent/multus/multus.go new file mode 100644 index 000000000000..c4f39793f531 --- /dev/null +++ b/pkg/agent/multus/multus.go @@ -0,0 +1,128 @@ +package multus + +// Generate a code which creates a kubernetes Job object with an image +// and uses a selector to only deploy in a node, whose name can be be picked up from the config AgentConfig.NodeName + +import ( + "context" + "fmt" + "time" + + "github.com/k3s-io/k3s/pkg/daemons/config" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +func StartMultusJob(ctx context.Context, nodeConfig *config.Node) error { + imageName, err := fetchImageName(ctx, nodeConfig) + if err != nil { + return err + } + + fmt.Printf("MANU - This is the imageName: %v\n", imageName) + + job := generateJob(nodeConfig.AgentConfig.NodeName, imageName, nodeConfig.AgentConfig.CNIBinDir) + + fmt.Printf("MANU - This is the job: %v\n", job) + config, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) + if err != nil { + return err + } + + // Create a new clientset + clientset, _ := kubernetes.NewForConfig(config) + _, err = clientset.BatchV1().Jobs("kube-system").Create(context.TODO(), &job, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil +} + +// fetchImageName() is a function that returns the image name by using helm and reading the initContainer of the multus chart +func fetchImageName(ctx context.Context, nodeConfig *config.Node) (string, error) { + config, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) + if err != nil { + return "", err + } + + // Create a new clientset + clientset, _ := kubernetes.NewForConfig(config) + daemonset, _ := clientset.AppsV1().DaemonSets("kube-system").Get(context.TODO(), "multus", metav1.GetOptions{}) + + var imageName string + if err := wait.PollImmediateWithContext(ctx, 5*time.Second, 100*time.Second, func(ctx context.Context) (bool, error) { + for _, initContainer := range daemonset.Spec.Template.Spec.InitContainers { + if initContainer.Name == "cni-plugins" { + imageName = initContainer.Image + fmt.Printf("MANU - ImageName found! %v\n", imageName) + return true, nil + } + } + fmt.Printf("MANU - Could not find the multus initContainer image, trying again in 10 seconds\n") + return false, nil + }); err != nil { + return "", fmt.Errorf("time out trying to find the multus initContainer image") + } + + return imageName, nil +} + +func generateJob(nodeName, imageName, cniBinDir string) batchv1.Job { + hostPathDirectoryOrCreate := corev1.HostPathDirectoryOrCreate + // Create a new Job object + job := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multus", + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multus", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "multus", + Image: imageName, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/host/opt/cni/bin", + Name: "cni-path", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "SKIP_CNI_BINARIES", + Value: "flannel", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "cni-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: cniBinDir, + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + }, + NodeSelector: map[string]string{ + "kubernetes.io/hostname": nodeName, + }, + }, + }, + }, + } + + return job +} \ No newline at end of file diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 19bd38ab51ca..f1bf70e49db2 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -15,6 +15,7 @@ import ( "github.com/k3s-io/k3s/pkg/agent/config" "github.com/k3s-io/k3s/pkg/agent/containerd" "github.com/k3s-io/k3s/pkg/agent/flannel" + "github.com/k3s-io/k3s/pkg/agent/multus" "github.com/k3s-io/k3s/pkg/agent/netpol" "github.com/k3s-io/k3s/pkg/agent/proxy" "github.com/k3s-io/k3s/pkg/agent/syssetup" @@ -129,6 +130,12 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { if err := flannel.Prepare(ctx, nodeConfig); err != nil { return err } + + if nodeConfig.Multus { + if err := multus.StartMultusJob(ctx, nodeConfig); err != nil { + return err + } + } } if nodeConfig.Docker { diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 592cc1a568b2..dc930458bf8c 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -40,6 +40,7 @@ type Node struct { ContainerRuntimeEndpoint string ImageServiceEndpoint string NoFlannel bool + Multus bool SELinux bool EmbeddedRegistry bool FlannelBackend string @@ -157,7 +158,7 @@ type CriticalControlArgs struct { FlannelIPv6Masq bool `cli:"flannel-ipv6-masq"` FlannelExternalIP bool `cli:"flannel-external-ip"` EgressSelectorMode string `cli:"egress-selector-mode"` - Multus bool `cli:"multus"` + Multus bool `cli:"multus"` ServiceIPRange *net.IPNet `cli:"service-cidr"` ServiceIPRanges []*net.IPNet `cli:"service-cidr"` } diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 8a3783a3b849..95f1354013aa 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -313,7 +313,7 @@ func multusYaml() (*asset, error) { return a, nil } -var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x94\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x0a\x21\xbb\x72\x38\xdc\x72\xf0\xd8\x0e\xdd\x03\xb4\xbb\x2c\xb1\x09\x6b\x59\x14\x48\x39\x41\xfb\xeb\x0b\xc7\x4e\xd2\xc4\x76\xe0\xb4\xe9\x66\x0b\xe2\xfb\x48\xbe\x07\xd9\x84\x2f\xc0\x82\x14\x0b\xcd\xa5\x75\x4b\xdb\xe4\x0d\x31\x7e\xd8\x8c\x14\x97\xd5\x7f\x59\x22\xfd\xd9\xfe\x55\x15\x46\x5f\xe8\xc7\xd0\x48\x06\x5e\x51\x80\x07\x8c\x1e\xe3\x5a\xd5\x90\xad\xb7\xd9\x16\x4a\xeb\x68\x6b\x28\x74\xd5\x94\x60\x6c\x42\x01\xde\x02\x9b\xf6\x37\x40\x36\xd6\xd7\x18\x15\x53\x80\x15\xbc\xb6\xb7\x6d\xc2\x27\xa6\x26\x5d\x21\x2b\xad\x07\xe0\x23\x47\xde\x25\x43\x5d\x1c\xf5\x13\xf6\x0c\x69\xca\x37\x70\x59\x0a\x65\x6e\x82\x3c\x0b\xf0\xc4\x14\x4a\x19\x63\xd4\xf7\xb7\x35\xb2\xa6\x43\xfb\xff\xc4\x38\x8a\x99\x29\x04\x60\xc5\x4d\x80\xb3\xc6\xa5\xad\x30\x7a\xb1\x50\x5a\x33\x08\x35\xec\xa0\x3f\x8b\xe4\x41\x94\xd6\x5b\xe0\xb2\x3f\x5a\x43\x9e\x59\x6b\x6b\x90\x64\xdd\xa5\x40\x40\xc9\xfb\x8f\x9d\xcd\x6e\x33\xa2\x15\x21\xef\x88\x2b\x8c\xeb\x7e\xde\x31\xf1\xee\x4e\xa2\x80\x0e\xf7\x04\xa3\x5d\xb7\x0c\x87\x9e\x6f\x45\x8e\x10\x20\xfa\x44\x18\x73\xa7\x9d\xc8\x4f\x69\xb6\x0b\x39\x69\xff\xd0\xc5\xe9\xcc\x4f\x98\x79\xff\xb0\x9f\x03\x4e\x49\x6f\x67\x9c\xc7\xb8\x48\xfb\x75\xc0\xfd\x63\xff\x35\x07\xa6\x4d\xf0\x64\xe4\x07\x49\x1b\xc6\x60\x76\xa8\x7e\xcd\xf8\x91\x71\xee\x67\xfa\x50\xfc\xdc\xf0\xae\x72\x8f\x18\x3a\x79\x78\x1d\xe6\xb5\xf1\x19\x00\x00\xff\xff\x20\xa2\xda\xb0\x09\x06\x00\x00") +var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x94\x4f\x8f\xd4\x30\x0c\xc5\xef\xf9\x14\x56\xef\x19\x84\xb8\xa0\x1e\xe1\xc0\x7d\x25\xb8\x8c\xf6\x90\x26\x66\xd6\x3b\x69\x1c\xd9\xe9\xac\xe0\xd3\xa3\xfe\xd9\x85\x6e\xdb\x55\x07\x86\xdb\x8c\x95\xbc\x5f\x6c\xbf\x57\x97\xe9\x1b\x8a\x12\xa7\x1a\xa4\x71\xfe\xe0\xba\xf2\xc0\x42\x3f\x5d\x21\x4e\x87\xf3\x47\x3d\x10\xbf\xbb\xbc\x37\x67\x4a\xa1\x86\xcf\xb1\xd3\x82\x72\xc7\x11\x3f\x51\x0a\x94\x4e\xa6\xc5\xe2\x82\x2b\xae\x36\x00\xc9\xb5\x58\xc3\xb9\x6b\xd0\xba\x4c\x8a\x72\x41\xb1\xfd\xdf\x88\xc5\xba\xd0\x52\x32\xc2\x11\xef\xf0\x7b\x7f\xda\x65\xfa\x22\xdc\xe5\x37\xc8\x06\x60\x01\x7e\xe1\xe8\x0f\x2d\xd8\xd6\x2f\xfa\x99\x26\x86\x76\xcd\x23\xfa\xa2\xb5\xb1\x57\x41\xbe\x2a\xca\x46\x17\xc6\x58\x6b\xcd\xdf\x4f\x6b\x65\x4c\xcf\xcf\xff\xa0\xd6\x73\x2a\xc2\x31\xa2\x18\xe9\x22\xce\x1e\xae\xfd\x0d\x0b\x55\x65\x00\x04\x95\x3b\xf1\x38\xd5\x12\x07\x54\x03\x70\x41\x69\xa6\xd2\x09\xcb\xce\xbb\xae\x45\xcd\xce\xbf\x16\x88\xa4\x65\xf8\xf1\xe4\x8a\x7f\x58\xd1\x4a\x58\x9e\x58\xce\x94\x4e\x53\xbf\x6b\xe2\xe3\x99\xcc\x91\x3c\x0d\x04\x0b\x7e\x1c\x86\xa7\x20\xd7\x22\x57\x08\x98\x42\x66\x4a\x65\xd4\xce\x1c\xb6\x34\xfb\x81\xac\x6b\xc3\xb1\x6a\xfa\x62\x75\x3f\x93\x87\x63\xf5\xc8\x8d\x0e\xd5\x51\x10\x8e\x95\x17\x74\x05\xab\xfb\x7f\xb5\xc1\x76\x68\x36\xdc\x70\xfb\xb4\xcc\x01\xbf\xa3\xd2\x0f\x69\x1f\xe3\x55\x5c\xde\x06\xdc\x3e\x37\x7f\x1a\xc9\xf6\x11\xd8\xcc\xcc\xc2\xaa\x4b\x1f\xed\x76\xe5\x7f\x5b\xfc\x4a\x3b\xb7\x5b\xfa\x52\x7c\xbe\xf0\xf1\xe6\x80\x58\x6e\xf2\xf9\xf3\xb2\xef\x19\xbf\x02\x00\x00\xff\xff\x67\x80\xc4\xe2\x4a\x06\x00\x00") func rolebindingsYamlBytes() ([]byte, error) { return bindataRead(