Merge pull request #3157 from chrischdi/pr-kubadm-beta-control-plane-…

…local-kubelet-mode-v2 Update ControlPlaneKubeletLocalMode test for the feature gate being disabled and fix setup-external-ca in kinder
kubernetes · Feb 4, 2025 · 8e1f387 · 8e1f387
2 parents ae11614 + e9f1704
commit 8e1f387
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 43 deletions.
diff --git a/...ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml b/...ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -2,7 +2,7 @@
 version: 1
 summary: |
   This workflow implements a sequence of tasks used test the proper functioning
-  of the ControlPlaneKubeletLocalMode feature gate.
+  of having the ControlPlaneKubeletLocalMode feature gate set to false.
 vars:
   # vars defines default values for variable used by tasks in this workflow;
   # those values might be overridden when importing this files.
@@ -59,19 +59,19 @@ tasks:
   - --name={{ .vars.clusterName }}
   - --loglevel=debug
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
-  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
+  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=false"
   - --copy-certs=auto
   timeout: 5m
 - name: post-init
   description: |
     Run commands after kubeadm init is called on a primary CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to load balanced apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
 
       # Ensure kubelet.conf points to the local IP.
@@ -95,22 +95,23 @@ tasks:
 - name: post-join
   description: |
     Run commands after kubeadm join is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
@@ -119,23 +120,24 @@ tasks:
 - name: pre-upgrade
   description: |
     Run commands before kubeadm upgrade is called on all joined CP node to replace
-    the server in the kubelet's kubeconfig to point to the load balancer.
+    the server in the kubelet's kubeconfig to point to the control-plane IP.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)
-
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
       exit 0
@@ -155,22 +157,21 @@ tasks:
 - name: post-upgrade
   description: |
     Run commands after kubeadm upgrade is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 

diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml
@@ -1,7 +1,7 @@
 version: 1
 summary: |
-  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using
-  the ControlPlaneKubeletLocalMode feature gate.
+  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes having
+  the ControlPlaneKubeletLocalMode feature gate set to false.
   test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-mode{{ dashVer .KubernetesVersion }}
   config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
 vars:

diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml
@@ -1,8 +1,8 @@
 # AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
 version: 1
 summary: |
-  This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using
-  the ControlPlaneKubeletLocalMode feature gate.
+  This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes having
+  the ControlPlaneKubeletLocalMode feature gate set to false.
   test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-modelatest
   config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-control-plane-local-kubelet-mode.yaml
 vars:

diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml
@@ -3,7 +3,7 @@
 version: 1
 summary: |
   This workflow implements a sequence of tasks used test the proper functioning
-  of the ControlPlaneKubeletLocalMode feature gate.
+  of having the ControlPlaneKubeletLocalMode feature gate set to false.
 vars:
   # vars defines default values for variable used by tasks in this workflow;
   # those values might be overridden when importing this files.
@@ -60,19 +60,19 @@ tasks:
   - --name={{ .vars.clusterName }}
   - --loglevel=debug
   - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
-  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true"
+  - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=false"
   - --copy-certs=auto
   timeout: 5m
 - name: post-init
   description: |
     Run commands after kubeadm init is called on a primary CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to load balanced apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
 
       # Ensure kubelet.conf points to the local IP.
@@ -96,22 +96,23 @@ tasks:
 - name: post-join
   description: |
     Run commands after kubeadm join is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
+
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
 
-      # Ensure kubelet.conf points to the local IP.
+      # Ensure kubelet.conf points to the remote IP.
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
@@ -120,23 +121,24 @@ tasks:
 - name: pre-upgrade
   description: |
     Run commands before kubeadm upgrade is called on all joined CP node to replace
-    the server in the kubelet's kubeconfig to point to the load balancer.
+    the server in the kubelet's kubeconfig to point to the control-plane IP.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)
-
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
-      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
+      ${CMD} sed -i 's@server: https://.*:6443@server: https://'${IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1
 
       # Ensure exit status of 0
       exit 0
@@ -156,22 +158,21 @@ tasks:
 - name: post-upgrade
   description: |
     Run commands after kubeadm upgrade is called on all joined CP node to checks if
-    the kubelet's kubeconfig file points to the local apiserver.
+    the kubelet's kubeconfig file points to the remote apiserver.
   cmd: /bin/bash
   args:
     - -c
     - |
       set -x
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)"
+      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb)"
+
       CMD="docker exec {{ .vars.clusterName }}-control-plane-1"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-2"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 
-      IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)"
       CMD="docker exec {{ .vars.clusterName }}-control-plane-3"
       ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1
 

diff --git a/kinder/pkg/cluster/manager/actions/setup-external-ca.go b/kinder/pkg/cluster/manager/actions/setup-external-ca.go
@@ -72,6 +72,18 @@ func SetupExternalCA(c *status.Cluster, vLevel int) error {
 		return nil
 	}
 
+	generateKubeletConfWorker := func(n *status.Node) error {
+		if err := n.Command(
+			"/bin/sh", "-c",
+			fmt.Sprintf("kubeadm init phase kubeconfig kubelet --control-plane-endpoint=%s --apiserver-advertise-address=%s --v=%d",
+				loadBalancerIP, loadBalancerIP,
+				vLevel),
+		).RunWithEcho(); err != nil {
+			return errors.Wrapf(err, "could not generate a kubelet.conf on node: %s", n.Name())
+		}
+		return nil
+	}
+
 	// iterate secondary CP nodes
 	for _, n := range c.SecondaryControlPlanes() {
 		// copy the shared kubeconfig files
@@ -108,7 +120,7 @@ func SetupExternalCA(c *status.Cluster, vLevel int) error {
 		}
 
 		// generate kubelet.conf
-		if err := generateKubeletConf(n); err != nil {
+		if err := generateKubeletConfWorker(n); err != nil {
 			return err
 		}
 	}