Implement OS Version reconcile

Signed-off-by: Andrea Mazzotti <[email protected]>

api/v1beta1/constants.go

@@ -35,6 +35,9 @@ const (
 	LabelElementalHostBootstrapped         = "elementalhost.infrastructure.cluster.x-k8s.io/bootstrapped"
 	LabelElementalHostNeedsReset           = "elementalhost.infrastructure.cluster.x-k8s.io/needs-reset"
 	LabelElementalHostReset                = "elementalhost.infrastructure.cluster.x-k8s.io/reset"
+	LabelElementalHostInPlaceUpgrade       = "elementalhost.infrastructure.cluster.x-k8s.io/in-place-upgrade"
+	InPlaceUpgradePending                  = "pending"
+	InPlaceUpgradeDone                     = "done"
 )
 
 // HostPhases.
@@ -48,6 +51,7 @@ const (
 	PhaseRunning                = HostPhase("Running")
 	PhaseTriggeringReset        = HostPhase("Triggering Reset")
 	PhaseResetting              = HostPhase("Resetting")
+	PhaseOSVersionReconcile     = HostPhase("Reconciling OS Version")
 )
 
 // Conditions.
@@ -88,6 +92,14 @@ const (
 	WaitingForResetReasonSeverity clusterv1.ConditionSeverity = clusterv1.ConditionSeverityInfo
 	// ResetFailedReason indicates that the Host reset failed.
 	ResetFailedReason = "ResetFailed"
+
+	// OSVersionReady describes the Host OS version reconciliation phase.
+	OSVersionReady clusterv1.ConditionType = "OSVersionReady"
+	// OSVersionReconciliationFailedReason indicates that the attempted Host OS version reconciliation failed.
+	OSVersionReconciliationFailedReason = "OSVersionReconciliationFailed"
+	// WaitingForPostReconcileRebootReason indicates that the Host OS version was applied and the Host is going to reboot.
+	WaitingForPostReconcileRebootReason                                     = "WaitingForPostReconcileReboot"
+	WaitingForPostReconcileRebootReasonSeverity clusterv1.ConditionSeverity = clusterv1.ConditionSeverityInfo
 )
 
 // ElementalMachine Conditions and Reasons.

api/v1beta1/elementalhost_types.go

@@ -19,6 +19,7 @@ package v1beta1
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
@@ -35,6 +36,13 @@ type ElementalHostSpec struct {
 	// PubKey is the host public key to verify when authenticating
 	// Elemental API requests for this host.
 	PubKey string `json:"pubKey,omitempty"`
+	// OSVersionManagement defines the OS Version and options to be reconciled
+	// on the host. The supported schema depends on the OSPlugin in use by
+	// the elementa-agent.
+	// +optional
+	// +kubebuilder:validation:Schemaless
+	// +kubebuilder:validation:XPreserveUnknownFields
+	OSVersionManagement map[string]runtime.RawExtension `json:"osVersionManagement,omitempty" yaml:"osVersionManagement,omitempty"`
 }
 
 // ElementalHostStatus defines the observed state of ElementalHost.

api/v1beta1/elementalmachine_types.go

@@ -19,6 +19,7 @@ package v1beta1
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
@@ -37,6 +38,15 @@ type ElementalMachineSpec struct {
 	// using this host.
 	// +optional
 	HostRef *corev1.ObjectReference `json:"hostRef,omitempty"`
+
+	// OSVersionManagement defines the OS Version and options to be reconciled
+	// on the host. The supported schema depends on the OSPlugin in use by
+	// the elementa-agent. Whenever an ElementalHost is associated to this
+	// ElementalMachine, the OSVersionManagement will be applied to it.
+	// +optional
+	// +kubebuilder:validation:Schemaless
+	// +kubebuilder:validation:XPreserveUnknownFields
+	OSVersionManagement map[string]runtime.RawExtension `json:"osVersionManagement,omitempty" yaml:"osVersionManagement,omitempty"`
 }
 
 // ElementalMachineStatus defines the observed state of ElementalMachine.

cmd/agent/common.go

@@ -0,0 +1,26 @@
+package agent
+
+import (
+	infrastructurev1 "github.com/rancher-sandbox/cluster-api-provider-elemental/api/v1beta1"
+	"github.com/rancher-sandbox/cluster-api-provider-elemental/internal/agent/log"
+	"github.com/rancher-sandbox/cluster-api-provider-elemental/pkg/agent/osplugin"
+)
+
+// handlePost handles post conditions such as Reboot or PowerOff.
+// A true flag is returned if any of the conditions is true, to highlight the program should exit.
+func handlePost(osPlugin osplugin.Plugin, post infrastructurev1.PostAction) bool {
+	if post.PowerOff {
+		log.Info("Powering off system")
+		if err := osPlugin.PowerOff(); err != nil {
+			log.Error(err, "Powering off system")
+		}
+		return true
+	} else if post.Reboot {
+		log.Info("Rebooting system")
+		if err := osPlugin.Reboot(); err != nil {
+			log.Error(err, "Rebooting system")
+		}
+		return true
+	}
+	return false
+}

cmd/agent/run.go

@@ -7,7 +7,6 @@ import (
 	"github.com/rancher-sandbox/cluster-api-provider-elemental/internal/agent/log"
 	"github.com/rancher-sandbox/cluster-api-provider-elemental/internal/agent/phase"
 	"github.com/rancher-sandbox/cluster-api-provider-elemental/internal/api"
-	"github.com/rancher-sandbox/cluster-api-provider-elemental/pkg/agent/osplugin"
 	"github.com/spf13/cobra"
 )
 
@@ -60,6 +59,23 @@ This command will reconcile the remote ElementalHost resource describing this ho
 				return
 			}
 
+			// Handle Upgrade
+			if !host.Bootstrapped || host.InPlaceUpgrade == infrastructurev1.InPlaceUpgradePending {
+				log.Info("Reconciling OS Version")
+				osVersionHandler := phase.NewOSVersionHandler(*agentContext)
+				post, err := osVersionHandler.Reconcile(host.OSVersionManagement)
+				if err != nil {
+					log.Error(err, "handling OS reconciliation")
+					log.Debugf("Waiting %s...", agentContext.Config.Agent.Reconciliation.String())
+					time.Sleep(agentContext.Config.Agent.Reconciliation)
+					continue
+				}
+				if handlePost(agentContext.Plugin, post) {
+					// Exit the program if we are rebooting to apply bootstrap
+					return
+				}
+			}
+
 			// Handle bootstrap if needed
 			if host.BootstrapReady && !host.Bootstrapped {
 				log.Info("Handling bootstrap application")
@@ -86,22 +102,3 @@ This command will reconcile the remote ElementalHost resource describing this ho
 func init() {
 	rootCmd.AddCommand(runCmd)
 }
-
-// handlePost handles post conditions such as Reboot or PowerOff.
-// A true flag is returned if any of the conditions is true, to highlight the program should exit.
-func handlePost(osPlugin osplugin.Plugin, post infrastructurev1.PostAction) bool {
-	if post.PowerOff {
-		log.Info("Powering off system")
-		if err := osPlugin.PowerOff(); err != nil {
-			log.Error(err, "Powering off system")
-		}
-		return true
-	} else if post.Reboot {
-		log.Info("Rebooting system")
-		if err := osPlugin.Reboot(); err != nil {
-			log.Error(err, "Rebooting system")
-		}
-		return true
-	}
-	return false
-}

config/crd/bases/infrastructure.cluster.x-k8s.io_elementalhosts.yaml

@@ -154,6 +154,12 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
+              osVersionManagement:
+                description: |-
+                  OSVersionManagement defines the OS Version and options to be reconciled
+                  on the host. The supported schema depends on the OSPlugin in use by
+                  the elementa-agent.
+                x-kubernetes-preserve-unknown-fields: true
               pubKey:
                 description: |-
                   PubKey is the host public key to verify when authenticating

config/crd/bases/infrastructure.cluster.x-k8s.io_elementalmachines.yaml

@@ -105,6 +105,13 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
+              osVersionManagement:
+                description: |-
+                  OSVersionManagement defines the OS Version and options to be reconciled
+                  on the host. The supported schema depends on the OSPlugin in use by
+                  the elementa-agent. Whenever an ElementalHost is associated to this
+                  ElementalMachine, the OSVersionManagement will be applied to it.
+                x-kubernetes-preserve-unknown-fields: true
               providerID:
                 description: |-
                   ProviderID references the associated ElementalHost.

config/crd/bases/infrastructure.cluster.x-k8s.io_elementalmachinetemplates.yaml

@@ -101,6 +101,13 @@ spec:
                             type: string
                         type: object
                         x-kubernetes-map-type: atomic
+                      osVersionManagement:
+                        description: |-
+                          OSVersionManagement defines the OS Version and options to be reconciled
+                          on the host. The supported schema depends on the OSPlugin in use by
+                          the elementa-agent. Whenever an ElementalHost is associated to this
+                          ElementalMachine, the OSVersionManagement will be applied to it.
+                        x-kubernetes-preserve-unknown-fields: true
                       providerID:
                         description: |-
                           ProviderID references the associated ElementalHost.

doc/QUICKSTART.md

@@ -20,15 +20,14 @@
 
     ```bash
     # Install dependencies
-    zypper install -y docker helm kubernetes1.27-client
+    zypper install -y docker helm kubernetes-client make yq
 
     # Install kind
-    [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
-    chmod +x ./kind
-    mv ./kind /usr/local/bin/kind
+    curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
+    install -o root -g root -m 0755 kind /usr/local/bin/kind
 
     # Install clusterctl
-    curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.3/clusterctl-linux-amd64 -o clusterctl
+    curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.1/clusterctl-linux-amd64 -o clusterctl
     install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl
 
     systemctl enable docker
@@ -48,7 +47,7 @@
     apiVersion: kind.x-k8s.io/v1alpha4
     nodes:
     - role: control-plane
-      image: kindest/node:v1.26.6
+      image: kindest/node:v1.31.0
       kubeadmConfigPatches:
       - |
         kind: InitConfiguration

elemental-openapi.yaml

@@ -337,6 +337,9 @@ components:
           type: boolean
         condition:
           $ref: '#/components/schemas/V1Beta1Condition'
+        inPlaceUpgrade:
+          nullable: true
+          type: string
         installed:
           nullable: true
           type: boolean
@@ -361,6 +364,8 @@ components:
           type: boolean
         bootstrapped:
           type: boolean
+        inPlaceUpgrade:
+          type: string
         installed:
           type: boolean
         labels:
@@ -371,6 +376,10 @@ components:
           type: string
         needsReset:
           type: boolean
+        osVersionManagement:
+          additionalProperties:
+            $ref: '#/components/schemas/RuntimeRawExtension'
+          type: object
       type: object
     ApiRegistrationResponse:
       properties:

internal/agent/elementalcli/runner.go

@@ -38,9 +38,16 @@ type Reset struct {
 	Debug           bool     `json:"debug,omitempty" mapstructure:"debug"`
 }
 
+type Upgrade struct {
+	ImageURI        string `json:"imageUri,omitempty" mapstructure:"imageUri"`
+	UpgradeRecovery bool   `json:"upgradeRecovery,omitempty" mapstructure:"upgradeRecovery"`
+	Debug           bool   `json:"debug,omitempty" mapstructure:"debug"`
+}
+
 type Runner interface {
 	Install(Install) error
 	Reset(Reset) error
+	Upgrade(Upgrade, string) error
 }
 
 func NewRunner() Runner {
@@ -103,6 +110,30 @@ func (r *runner) Reset(conf Reset) error {
 	return nil
 }
 
+func (r *runner) Upgrade(conf Upgrade, correlationID string) error {
+	log.Debug("Running elemental upgrade")
+	installerOpts := []string{"elemental"}
+	// There are no env var bindings in elemental-cli for elemental root options
+	// so root flags should be passed within the command line
+	if conf.Debug {
+		installerOpts = append(installerOpts, "--debug")
+	}
+	installerOpts = append(installerOpts, "upgrade")
+
+	cmd := exec.Command("elemental")
+	environmentVariables := mapToUpgradeEnv(conf, correlationID)
+	cmd.Env = append(os.Environ(), environmentVariables...)
+	cmd.Stdout = os.Stdout
+	cmd.Args = installerOpts
+	cmd.Stdin = os.Stdin
+	cmd.Stderr = os.Stderr
+	log.Debugf("running: %s\n with ENV:\n%s", strings.Join(installerOpts, " "), strings.Join(environmentVariables, "\n"))
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("running elemental upgrade: %w", err)
+	}
+	return nil
+}
+
 func mapToInstallEnv(conf Install) []string {
 	var variables []string
 	// See GetInstallKeyEnvMap() in https://github.com/rancher/elemental-toolkit/blob/main/pkg/constants/constants.go
@@ -130,6 +161,15 @@ func mapToResetEnv(conf Reset) []string {
 	return variables
 }
 
+func mapToUpgradeEnv(conf Upgrade, correlationID string) []string {
+	var variables []string
+	// See GetUpgradeKeyEnvMap() in https://github.com/rancher/elemental-toolkit/blob/main/pkg/constants/constants.go
+	variables = append(variables, formatEV("ELEMENTAL_UPGRADE_RECOVERY", strconv.FormatBool(conf.UpgradeRecovery)))
+	variables = append(variables, formatEV("ELEMENTAL_UPGRADE_SYSTEM", conf.ImageURI))
+	variables = append(variables, formatEV("ELEMENTAL_UPGRADE_CORRELATION_ID", correlationID))
+	return variables
+}
+
 func formatEV(key string, value string) string {
 	return fmt.Sprintf("%s=%s", key, value)
 }