K0sControlPlane update strategies

Signed-off-by: Alexey Makhov <[email protected]>
k0sproject · May 17, 2024 · e6f3522 · e6f3522
1 parent 6b623d2
commit e6f3522
Show file tree

Hide file tree

Showing 13 changed files with 568 additions and 95 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -230,4 +230,6 @@ jobs:
         with:
           name: smoketests-${{ matrix.smoke-suite }}-files
           path: |
-            /tmp/${{ matrix.smoke-suite }}-k0smotron.log
+            /tmp/${{ matrix.smoke-suite }}-k0smotron-bootstrap.log
+            /tmp/${{ matrix.smoke-suite }}-k0smotron-control-plane.log
+            /tmp/${{ matrix.smoke-suite }}-k0smotron-infrastructure.log
diff --git a/api/controlplane/v1beta1/k0s_types.go b/api/controlplane/v1beta1/k0s_types.go
@@ -47,6 +47,10 @@ type K0sControlPlaneSpec struct {
 	//+kubebuilder:validation:Optional
 	//+kubebuilder:default=1
 	Replicas int32 `json:"replicas,omitempty"`
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:validation:Enum:rollout;inplace
+	//+kubebuilder:default=inplace
+	UpdateStrategy string `json:"updateStrategy,omitempty"`
 	// Version defines the k0s version to be deployed. You can use a specific k0s version (e.g. v1.27.1+k0s.0) or
 	// just the Kubernetes version (e.g. v1.27.1). If left empty, k0smotron will select one automatically.
 	//+kubebuilder:validation:Optional

diff --git a/config/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml b/config/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml
@@ -206,6 +206,9 @@ spec:
                 default: 1
                 format: int32
                 type: integer
+              updateStrategy:
+                default: inplace
+                type: string
               version:
                 description: |-
                   Version defines the k0s version to be deployed. You can use a specific k0s version (e.g. v1.27.1+k0s.0) or

diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml
@@ -206,6 +206,9 @@ spec:
                 default: 1
                 format: int32
                 type: integer
+              updateStrategy:
+                default: inplace
+                type: string
               version:
                 description: |-
                   Version defines the k0s version to be deployed. You can use a specific k0s version (e.g. v1.27.1+k0s.0) or

diff --git a/docs/resource-reference.md b/docs/resource-reference.md
@@ -1180,6 +1180,15 @@ Resource Types:
             <i>Default</i>: 1<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>updateStrategy</b></td>
+        <td>string</td>
+        <td>
+          <br/>
+          <br/>
+            <i>Default</i>: inplace<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>version</b></td>
         <td>string</td>

diff --git a/internal/controller/bootstrap/controlplane_bootstrap_controller.go b/internal/controller/bootstrap/controlplane_bootstrap_controller.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
@@ -40,6 +41,7 @@ import (
 	"sigs.k8s.io/cluster-api/util"
 	capiutil "sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
+	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -179,14 +181,24 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, fmt.Errorf("control plane endpoint is not set")
 	}
 
-	if strings.HasSuffix(config.Name, "-0") {
+	machines, err := collections.GetFilteredMachinesForCluster(ctx, c.Client, cluster, collections.ControlPlaneMachines(cluster.Name), collections.ActiveMachines)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("error collecting machines: %w", err)
+	}
+
+	if machines.Oldest().Name == config.Name {
 		files, err = c.genInitialControlPlaneFiles(ctx, scope, files)
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("error generating initial control plane files: %v", err)
 		}
 		installCmd = createCPInstallCmd(config)
 	} else {
-		files, err = c.genControlPlaneJoinFiles(ctx, scope, config, files)
+		oldest := getFirstRunningMachineWithLatestVersion(machines)
+		if oldest == nil {
+			return ctrl.Result{}, fmt.Errorf("wait for initial control plane provisioning")
+		}
+		files, err = c.genControlPlaneJoinFiles(ctx, scope, files, oldest)
+		//files, err = c.genControlPlaneJoinFiles(ctx, scope, config, files)
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("error generating control plane join files: %v", err)
 		}
@@ -289,7 +301,7 @@ func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context
 	return files, nil
 }
 
-func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *Scope, config *bootstrapv1.K0sControllerConfig, files []cloudinit.File) ([]cloudinit.File, error) {
+func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *Scope, files []cloudinit.File, firstControllerMachine *clusterv1.Machine) ([]cloudinit.File, error) {
 	log := log.FromContext(ctx).WithValues("K0sControllerConfig cluster", scope.Cluster.Name)
 
 	_, ca, err := c.getCerts(ctx, scope)
@@ -316,7 +328,7 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s
 		return nil, err
 	}
 
-	host, err := c.findFirstControllerIP(ctx, config)
+	host, err := c.findFirstControllerIP(ctx, firstControllerMachine)
 	if err != nil {
 		log.Error(err, "Failed to get controller IP")
 		return nil, err
@@ -526,22 +538,9 @@ func createCPInstallCmdWithJoinToken(config *bootstrapv1.K0sControllerConfig, to
 	return strings.Join(installCmd, " ")
 }
 
-func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, config *bootstrapv1.K0sControllerConfig) (string, error) {
-	// Dirty first controller name generation
-	nameParts := strings.Split(config.Name, "-")
-	nameParts[len(nameParts)-1] = "0"
-	name := strings.Join(nameParts, "-")
-	machine, machineImpl, err := c.getMachineImplementation(ctx, name, config)
-	if err != nil {
-		return "", fmt.Errorf("error getting machine implementation: %w", err)
-	}
-	addresses, found, err := unstructured.NestedSlice(machineImpl.UnstructuredContent(), "status", "addresses")
-	if err != nil {
-		return "", err
-	}
-
+func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, firstControllerMachine *clusterv1.Machine) (string, error) {
 	extAddr, intAddr := "", ""
-	for _, addr := range machine.Status.Addresses {
+	for _, addr := range firstControllerMachine.Status.Addresses {
 		if addr.Type == clusterv1.MachineExternalIP {
 			extAddr = addr.Address
 			break
@@ -552,16 +551,29 @@ func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, conf
 		}
 	}
 
-	if found {
-		for _, addr := range addresses {
-			addrMap, _ := addr.(map[string]interface{})
-			if addrMap["type"] == string(v1.NodeExternalIP) {
-				extAddr = addrMap["address"].(string)
-				break
-			}
-			if addrMap["type"] == string(v1.NodeInternalIP) {
-				intAddr = addrMap["address"].(string)
-				break
+	name := firstControllerMachine.Name
+
+	if extAddr == "" && intAddr == "" {
+		machineImpl, err := c.getMachineImplementation(ctx, firstControllerMachine)
+		if err != nil {
+			return "", fmt.Errorf("error getting machine implementation: %w", err)
+		}
+		addresses, found, err := unstructured.NestedSlice(machineImpl.UnstructuredContent(), "status", "addresses")
+		if err != nil {
+			return "", err
+		}
+
+		if found {
+			for _, addr := range addresses {
+				addrMap, _ := addr.(map[string]interface{})
+				if addrMap["type"] == string(v1.NodeExternalIP) {
+					extAddr = addrMap["address"].(string)
+					break
+				}
+				if addrMap["type"] == string(v1.NodeInternalIP) {
+					intAddr = addrMap["address"].(string)
+					break
+				}
 			}
 		}
 	}
@@ -577,13 +589,7 @@ func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, conf
 	return "", fmt.Errorf("no address found for machine %s", name)
 }
 
-func (c *ControlPlaneController) getMachineImplementation(ctx context.Context, name string, config *bootstrapv1.K0sControllerConfig) (*clusterv1.Machine, *unstructured.Unstructured, error) {
-	var machine clusterv1.Machine
-	err := c.Get(ctx, client.ObjectKey{Name: name, Namespace: config.Namespace}, &machine)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error getting machine object: %w", err)
-	}
-
+func (c *ControlPlaneController) getMachineImplementation(ctx context.Context, machine *clusterv1.Machine) (*unstructured.Unstructured, error) {
 	infRef := machine.Spec.InfrastructureRef
 
 	machineImpl := new(unstructured.Unstructured)
@@ -593,11 +599,11 @@ func (c *ControlPlaneController) getMachineImplementation(ctx context.Context, n
 
 	key := client.ObjectKey{Name: infRef.Name, Namespace: infRef.Namespace}
 
-	err = c.Get(ctx, key, machineImpl)
+	err := c.Get(ctx, key, machineImpl)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error getting machine implementation object: %w", err)
+		return nil, fmt.Errorf("error getting machine implementation object: %w", err)
 	}
-	return &machine, machineImpl, nil
+	return machineImpl, nil
 }
 
 func genShutdownServiceFiles() []cloudinit.File {
@@ -650,3 +656,31 @@ command="/etc/bin/k0sleave.sh"
 		},
 	}
 }
+
+func getFirstRunningMachineWithLatestVersion(machines collections.Machines) *clusterv1.Machine {
+	res := make(machinesByVersionAndCreationTimestamp, 0, len(machines))
+	for _, value := range machines {
+		if value.Status.Phase == string(clusterv1.MachinePhasePending) {
+			continue
+		}
+		res = append(res, value)
+	}
+	if len(res) == 0 {
+		return nil
+	}
+	sort.Sort(res)
+	return res[0]
+}
+
+// machinesByCreationTimestamp sorts a list of Machine by creation timestamp, using their names as a tie breaker.
+type machinesByVersionAndCreationTimestamp []*clusterv1.Machine
+
+func (o machinesByVersionAndCreationTimestamp) Len() int      { return len(o) }
+func (o machinesByVersionAndCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
+func (o machinesByVersionAndCreationTimestamp) Less(i, j int) bool {
+
+	if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) {
+		return o[i].Name < o[j].Name
+	}
+	return *o[i].Spec.Version < *o[j].Spec.Version && o[i].CreationTimestamp.Before(&o[j].CreationTimestamp)
+}