diff --git a/controllers/machineset_controller.go b/controllers/machineset_controller.go new file mode 100644 index 0000000..6cb3342 --- /dev/null +++ b/controllers/machineset_controller.go @@ -0,0 +1,150 @@ +package controllers + +import ( + "context" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + csv1beta1 "github.com/appuio/machine-api-provider-cloudscale/api/cloudscale/provider/v1beta1" +) + +// MachineSetReconciler reconciles a MachineSet object +type MachineSetReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +const ( + // This exposes compute information based on the providerSpec input. + // This is needed by the autoscaler to foresee upcoming capacity when scaling from zero. + // https://github.com/openshift/enhancements/pull/186 + cpuKey = "machine.openshift.io/vCPU" + memoryKey = "machine.openshift.io/memoryMb" + gpuKey = "machine.openshift.io/GPU" + labelsKey = "capacity.cluster-autoscaler.kubernetes.io/labels" + + gpuKeyValue = "0" + arch = "kubernetes.io/arch=amd64" +) + +// Reconcile reacts to MachineSet changes and updates the annotations used by the OpenShift autoscaler. +// GPU is always set to 0, as cloudscale does not provide GPU instances. +// The architecture label is always set to amd64. +func (r *MachineSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var machineSet machinev1beta1.MachineSet + if err := r.Get(ctx, req.NamespacedName, &machineSet); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !machineSet.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + origSet := machineSet.DeepCopy() + + if machineSet.Annotations == nil { + machineSet.Annotations = make(map[string]string) + } + + spec, err := csv1beta1.ProviderSpecFromRawExtension(machineSet.Spec.Template.Spec.ProviderSpec.Value) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get provider spec from machine template: %w", err) + } + if spec == nil { + return ctrl.Result{}, nil + } + + flavour, err := parseCloudscaleFlavour(spec.Flavor) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to parse flavour %q: %w", spec.Flavor, err) + } + + machineSet.Annotations[cpuKey] = strconv.Itoa(flavour.CPU) + machineSet.Annotations[memoryKey] = strconv.Itoa(flavour.MemGB * 1024) + machineSet.Annotations[gpuKey] = gpuKeyValue + + // We guarantee that any existing labels provided via the capacity annotations are preserved. + // See https://github.com/kubernetes/autoscaler/pull/5382 and https://github.com/kubernetes/autoscaler/pull/5697 + machineSet.Annotations[labelsKey] = mergeCommaSeparatedKeyValuePairs( + arch, + machineSet.Annotations[labelsKey]) + + if equality.Semantic.DeepEqual(origSet.Annotations, machineSet.Annotations) { + return ctrl.Result{}, nil + } + + if err := r.Patch(ctx, &machineSet, client.MergeFrom(origSet)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to patch MachineSet %q: %w", machineSet.Name, err) + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *MachineSetReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&machinev1beta1.MachineSet{}). + Complete(r) +} + +type cloudscaleFlavour struct { + Type string + CPU int + MemGB int +} + +var cloudscaleFlavourRegexp = regexp.MustCompile(`^(\w+)-(\d+)-(\d+)$`) + +// Parse parses a cloudscale flavour string. +func parseCloudscaleFlavour(flavour string) (cloudscaleFlavour, error) { + parts := cloudscaleFlavourRegexp.FindStringSubmatch(flavour) + + if len(parts) != 4 { + return cloudscaleFlavour{}, fmt.Errorf("flavour %q does not match expected format", flavour) + } + cpu, err := strconv.Atoi(parts[2]) + if err != nil { + return cloudscaleFlavour{}, fmt.Errorf("failed to parse CPU from flavour %q: %w", flavour, err) + } + mem, err := strconv.Atoi(parts[3]) + if err != nil { + return cloudscaleFlavour{}, fmt.Errorf("failed to parse memory from flavour %q: %w", flavour, err) + } + + return cloudscaleFlavour{ + Type: parts[1], + CPU: cpu, + MemGB: mem, + }, nil +} + +// mergeCommaSeparatedKeyValuePairs merges multiple comma separated lists of key=value pairs into a single, comma-separated, list +// of key=value pairs. If a key is present in multiple lists, the value from the last list is used. +func mergeCommaSeparatedKeyValuePairs(lists ...string) string { + merged := make(map[string]string) + for _, list := range lists { + for _, kv := range strings.Split(list, ",") { + kv := strings.Split(kv, "=") + if len(kv) != 2 { + // ignore invalid key=value pairs + continue + } + merged[kv[0]] = kv[1] + } + } + // convert the map back to a comma separated list + var result []string + for k, v := range merged { + result = append(result, fmt.Sprintf("%s=%s", k, v)) + } + slices.Sort(result) + return strings.Join(result, ",") +} diff --git a/controllers/machineset_controller_test.go b/controllers/machineset_controller_test.go new file mode 100644 index 0000000..2e19e0d --- /dev/null +++ b/controllers/machineset_controller_test.go @@ -0,0 +1,64 @@ +package controllers + +import ( + "context" + "fmt" + "testing" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func Test_MachineSetReconciler_Reconcile(t *testing.T) { + ctx := context.Background() + + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, machinev1beta1.AddToScheme(scheme)) + + ms := &machinev1beta1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machineset1", + Namespace: "default", + Annotations: map[string]string{ + "random": "annotation", + labelsKey: "a=a,b=b", + }, + }, + Spec: machinev1beta1.MachineSetSpec{}, + } + + setFlavorOnMachineSet(ms, "plus-2-4") + + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(ms). + Build() + + subject := &MachineSetReconciler{ + Client: c, + Scheme: scheme, + } + + _, err := subject.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ms)}) + require.NoError(t, err) + updated := &machinev1beta1.MachineSet{} + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(ms), updated)) + assert.Equal(t, "2", updated.Annotations[cpuKey]) + assert.Equal(t, "4096", updated.Annotations[memoryKey]) + assert.Equal(t, "0", updated.Annotations[gpuKey]) + assert.Equal(t, "a=a,b=b,kubernetes.io/arch=amd64", updated.Annotations[labelsKey]) +} + +func setFlavorOnMachineSet(machine *machinev1beta1.MachineSet, flavor string) { + machine.Spec.Template.Spec.ProviderSpec.Value = &runtime.RawExtension{ + Raw: []byte(fmt.Sprintf(`{"flavor": "%s"}`, flavor)), + } +} diff --git a/main.go b/main.go index 64916ad..b3df9b3 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,7 @@ import ( "github.com/cloudscale-ch/cloudscale-go-sdk/v5" configv1 "github.com/openshift/api/config/v1" apifeatures "github.com/openshift/api/features" - machinev1 "github.com/openshift/api/machine/v1beta1" + machinev1beta1 "github.com/openshift/api/machine/v1beta1" "github.com/openshift/library-go/pkg/features" capimachine "github.com/openshift/machine-api-operator/pkg/controller/machine" "k8s.io/apimachinery/pkg/runtime" @@ -42,6 +42,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "github.com/appuio/machine-api-provider-cloudscale/controllers" "github.com/appuio/machine-api-provider-cloudscale/pkg/machine" ) @@ -53,7 +54,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(configv1.AddToScheme(scheme)) - utilruntime.Must(machinev1.AddToScheme(scheme)) + utilruntime.Must(machinev1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -173,6 +174,14 @@ func runManager(metricsAddr, probeAddr, watchNamespace string, enableLeaderElect os.Exit(1) } + if err := (&controllers.MachineSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "MachineSet") + os.Exit(1) + } + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1)