K0sControlPlane update strategies

Signed-off-by: Alexey Makhov <[email protected]>

.github/workflows/go.yml

@@ -159,6 +159,7 @@ jobs:
           - check-capi-controlplane-docker-worker
           - check-capi-remote-machine-template-update
           - check-capi-docker-machine-template-update
+          - check-capi-docker-machine-template-update-recreate
           - check-capi-remote-machine-job-provision
 
     uses: ./.github/workflows/capi-smoke-tests.yml

.github/workflows/prepare-build-env.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+
+set -eu
+
+goVersion="$(./vars.sh go_version)"
+golangciLintVersion="$(./vars.sh FROM=hack/tools golangci-lint_version)"
+pythonVersion="$(./vars.sh FROM=docs python_version)"
+
+cat <<EOF >>"$GITHUB_ENV"
+GO_VERSION=$goVersion
+GOLANGCI_LINT_VERSION=$golangciLintVersion
+PYTHON_VERSION=$pythonVersion
+EOF
+
+# shellcheck disable=SC1090
+. "$GITHUB_ENV"
+
+echo ::group::OS Environment
+env | sort
+echo ::endgroup::

api/controlplane/v1beta1/k0s_types.go

@@ -31,7 +31,7 @@ func init() {
 type UpdateStrategy string
 
 const (
-	UpdateInPlace UpdateStrategy = "InPlace"
+	UpdateInPlace  UpdateStrategy = "InPlace"
 	UpdateRecreate UpdateStrategy = "Recreate"
 )
 

internal/controller/bootstrap/providerid_controller.go

@@ -48,7 +48,8 @@ func (p *ProviderIDController) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	if machine.Spec.ProviderID == nil || *machine.Spec.ProviderID == "" {
-		return ctrl.Result{}, fmt.Errorf("waiting for providerID for the machine %s/%s", machine.Namespace, machine.Name)
+		log.Info("waiting for providerID for the machine " + machine.Name)
+		return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 	}
 
 	cluster, err := capiutil.GetClusterByName(ctx, p.Client, machine.Namespace, machine.Spec.ClusterName)

internal/controller/controlplane/helper.go

@@ -189,12 +189,20 @@ func (c *K0sController) markChildControlNodeToLeave(ctx context.Context, name st
 
 	err := clientset.RESTClient().
 		Patch(types.MergePatchType).
-		AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes/" + name).
-		Body([]byte(`{"metadata":{"annotations":{"k0smotron.io/leave":"true"}}}`)).
+		AbsPath("/apis/etcd.k0sproject.io/v1beta1/etcdmembers/" + name).
+		Body([]byte(`{"spec":{"leave":"true"}}`)).
 		Do(ctx).
 		Error()
-	if err != nil && !apierrors.IsNotFound(err) {
-		return fmt.Errorf("error marking control node to leave: %w", err)
+	if err != nil {
+		err := clientset.RESTClient().
+			Patch(types.MergePatchType).
+			AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes/" + name).
+			Body([]byte(`{"metadata":{"annotations":{"k0smotron.io/leave":"true"}}}`)).
+			Do(ctx).
+			Error()
+		if err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("error marking control node to leave: %w", err)
+		}
 	}
 
 	return nil

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -20,12 +20,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/Masterminds/semver"
 	"github.com/google/uuid"
 	autopilot "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
+	bootstrapv1 "github.com/k0sproject/k0smotron/api/bootstrap/v1beta1"
+	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
+	"github.com/k0sproject/k0smotron/internal/controller/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -46,10 +46,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-
-	bootstrapv1 "github.com/k0sproject/k0smotron/api/bootstrap/v1beta1"
-	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
-	"github.com/k0sproject/k0smotron/internal/controller/util"
+	"strings"
 )
 
 const (
@@ -230,10 +227,8 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 		replicasToReport = kcp.Status.Replicas
 	}
 
-	fmt.Println("asdfsadfsdafsdafa111")
-
 	if kcp.Status.Version != "" && kcp.Spec.Version != kcp.Status.Version {
-		if kcp.Spec.UpdateStrategy == "rollout" {
+		if kcp.Spec.UpdateStrategy == cpv1beta1.UpdateRecreate {
 			desiredReplicas += kcp.Spec.Replicas
 			machinesToDelete = int(kcp.Spec.Replicas)
 			replicasToReport = desiredReplicas
@@ -243,9 +238,10 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 				return replicasToReport, fmt.Errorf("error getting cluster client set for machine update: %w", err)
 			}
 
-		err = c.createAutopilotPlan(ctx, kcp, cluster, kubeClient)
-		if err != nil {
-			return replicasToReport, fmt.Errorf("error creating autopilot plan: %w", err)
+			err = c.createAutopilotPlan(ctx, kcp, cluster, kubeClient)
+			if err != nil {
+				return replicasToReport, fmt.Errorf("error creating autopilot plan: %w", err)
+			}
 		}
 	}
 
@@ -278,11 +274,6 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 		}
 	}
 
-	fmt.Println("asdfsadfsdafsdafa")
-	fmt.Println("machines", machines)
-	fmt.Println("machinesToDelete", machinesToDelete)
-
-	//var isNewMachineReady bool
 	for _, m := range machines {
 		ver := semver.MustParse(kcp.Spec.Version)
 		fmt.Println("machines ver", machinesToDelete, *m.Spec.Version, fmt.Sprintf("v%d.%d.%d", ver.Major(), ver.Minor(), ver.Patch()), m.Spec.Version != nil && *m.Spec.Version != fmt.Sprintf("v%d.%d.%d", ver.Major(), ver.Minor(), ver.Patch()))
@@ -291,13 +282,12 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 		}
 
 		if machinesToDelete > 0 {
-
 			kubeClient, err := c.getKubeClient(ctx, cluster)
 			if err != nil {
 				return replicasToReport, fmt.Errorf("error getting cluster client set for machine update: %w", err)
 			}
 			var cn autopilot.ControlNode
-			err = kubeClient.RESTClient().Get().AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes").Name(m.Name).Do(ctx).Into(&cn)
+			err = kubeClient.RESTClient().Get().AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes/" + m.Name).Do(ctx).Into(&cn)
 			fmt.Println("machines !!!", cn.Name, cn.Status)
 			if err != nil {
 				if apierrors.IsNotFound(err) {
@@ -340,7 +330,7 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 			return kcp.Status.Replicas, fmt.Errorf("waiting for previous machine to be deleted")
 		}
 
-		time.Sleep(time.Second * 10)
+		//time.Sleep(time.Second * 10)
 
 		replicasToReport -= 1
 		name := machine.Name

internal/util/dynamic_config.go

@@ -3,8 +3,11 @@ package util
 import (
 	"context"
 	"fmt"
+	"time"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util"
@@ -31,15 +34,19 @@ func ReconcileDynamicConfig(ctx context.Context, cluster metav1.Object, cli clie
 		return fmt.Errorf("failed to create workload cluster client: %w", err)
 	}
 
-	err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
+	err = retry.OnError(wait.Backoff{
+		Steps:    4,
+		Duration: 100 * time.Millisecond,
+		Factor:   5.0,
+		Jitter:   0.5,
+	}, func(err error) bool {
 		return true
 	}, func() error {
 		return chCS.Patch(ctx, u, client.RawPatch(client.Merge.Type(), b), []client.PatchOption{}...)
 	})
 	if err != nil {
 		return fmt.Errorf("failed to patch k0s config: %w", err)
 	}
-	//return chCS.Patch(ctx, u, client.RawPatch(client.Merge.Type(), b), []client.PatchOption{}...)
 
 	return nil
 }

inttest/Makefile

@@ -48,4 +48,5 @@ check-capi-remote-machine-template: TIMEOUT=12m
 check-capi-remote-machine-template-update: TIMEOUT=10m
 check-capi-docker-machine-template-update: TIMEOUT=10m
 check-capi-remote-machine-job-provision: TIMEOUT=10m
-check-capi-docker-machine-template-update-rollout: TIMEOUT=10m
+check-capi-docker-machine-template-update: TIMEOUT=10m
+check-capi-docker-machine-template-update-recreate: TIMEOUT=10m

inttest/Makefile.variables

@@ -30,4 +30,4 @@ smoketests := \
     check-capi-remote-machine-template \
     check-capi-remote-machine-template-update \
     check-capi-docker-machine-template-update \
-    check-capi-docker-machine-template-update-rollout \
+    check-capi-docker-machine-template-update-recreate \

inttest/capi-config-update-vm/capi_config_update_vm_test.go

@@ -105,9 +105,16 @@ func (s *CAPIConfigUpdateVMSuite) TestCAPIConfigUpdateVMWorker() {
 	s.Require().NoError(err)
 
 	s.T().Log("waiting for cluster to be ready")
+
+	var ids []string
 	// nolint:staticcheck
 	err = wait.PollImmediateUntilWithContext(s.ctx, 1*time.Second, func(ctx context.Context) (bool, error) {
-		output, err := exec.Command("docker", "exec", "docker-test-cluster-docker-test-0", "k0s", "status").Output()
+		ids, err = util.GetControlPlaneNodesIDs("docker-test-cluster-docker-test-0")
+		if err != nil {
+			return false, err
+		}
+
+		output, err := exec.Command("docker", "exec", ids[0], "k0s", "status").Output()
 		if err != nil {
 			return false, nil
 		}
@@ -118,7 +125,7 @@ func (s *CAPIConfigUpdateVMSuite) TestCAPIConfigUpdateVMWorker() {
 
 	// nolint:staticcheck
 	err = wait.PollImmediateUntilWithContext(s.ctx, 1*time.Second, func(ctx context.Context) (bool, error) {
-		output, err := exec.Command("docker", "exec", "docker-test-cluster-docker-test-0", "k0s", "kc", "--kubeconfig=/var/lib/k0s/pki/admin.conf", "get", "clusterconfig", "-A").Output()
+		output, err := exec.Command("docker", "exec", ids[0], "k0s", "kc", "--kubeconfig=/var/lib/k0s/pki/admin.conf", "get", "clusterconfig", "-A").Output()
 		if err != nil {
 			return false, nil
 		}