add: Machine Pool existence check (#102)

* update namespace for ocm secret

* add: machinePoolExists function to check if a machine pool is available to use

Co-authored-by: Fiona-Waters [email protected]

* add: update instascale-clusterrole.yaml, machinePoolExists and onAdd
func, check for instascale-ocm-secret before defaulting machinetype,
update error messages.

---------

Co-authored-by: Fiona Waters <[email protected]>

controllers/appWrapper_controller_test.go

@@ -86,7 +86,7 @@ func TestDiscoverInstanceTypes(t *testing.T) {
 			expected: map[string]int{},
 		},
 		{
-			name: "Test with empty orderedinstance",
+			name: "Test with no orderedinstance",
 			input: &arbv1.AppWrapper{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{

controllers/appwrapper_controller.go

@@ -45,8 +45,9 @@ import (
 // AppWrapperReconciler reconciles a AppWrapper object
 type AppWrapperReconciler struct {
 	client.Client
-	Scheme           *runtime.Scheme
-	ConfigsNamespace string
+	Scheme             *runtime.Scheme
+	ConfigsNamespace   string
+	OcmSecretNamespace string
 }
 
 var (
@@ -149,36 +150,49 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	instascaleConfigmap, err := kubeClient.CoreV1().ConfigMaps(r.ConfigsNamespace).Get(context.Background(), "instascale-config", metav1.GetOptions{})
 	if err != nil {
 		klog.Infof("Config map named instascale-config is not available in namespace %v", r.ConfigsNamespace)
+		return err
 	}
 
-	if maxScaleNodesAllowed, err = strconv.Atoi(instascaleConfigmap.Data["maxScaleoutAllowed"]); err != nil {
-		klog.Infof("Error converting %v to int. Setting maxScaleNodesAllowed to 3", maxScaleNodesAllowed)
+	maxScaleNodesAllowed, err = strconv.Atoi(instascaleConfigmap.Data["maxScaleoutAllowed"])
+	if err != nil {
+		klog.Warningf("Error converting %v to int. Setting maxScaleNodesAllowed to 3", maxScaleNodesAllowed)
 		maxScaleNodesAllowed = 3
 	}
-	klog.Infof("Got config map named %v from namespace %v that configures max nodes in cluster to value %v", instascaleConfigmap.Name, instascaleConfigmap.Namespace, maxScaleNodesAllowed)
+	if instascaleConfigmap != nil {
+		klog.Errorf("Got config map named %v from namespace %v that configures max nodes in cluster to value %v", instascaleConfigmap.Name, instascaleConfigmap.Namespace, maxScaleNodesAllowed)
+	}
 
 	useMachineSets = true
-	useMachinePools, err := strconv.ParseBool(instascaleConfigmap.Data["useMachinePools"])
-	if err != nil {
-		klog.Infof("Error converting %v to bool. Defaulting to using Machine Sets", useMachineSets)
-	} else {
-		useMachineSets = !useMachinePools
-		klog.Infof("Setting useMachineSets to %v", useMachineSets)
-	}
+	ocmSecretExists := ocmSecretExists(r.OcmSecretNamespace)
+	if ocmSecretExists {
+		machinePoolsExists := machinePoolExists()
 
-	if !useMachineSets {
-		instascaleOCMSecret, err := kubeClient.CoreV1().Secrets(r.ConfigsNamespace).Get(context.Background(), "instascale-ocm-secret", metav1.GetOptions{})
-		if err != nil {
-			klog.Errorf("Error getting instascale-ocm-secret from namespace %v - Error :  %v", r.ConfigsNamespace, err)
+		if machinePoolsExists {
+			useMachineSets = false
+			klog.Infof("Using machine pools %v", machinePoolsExists)
+		} else {
+			klog.Infof("Setting useMachineSets to %v", useMachineSets)
 		}
-		ocmToken = string(instascaleOCMSecret.Data["token"])
 	}
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&arbv1.AppWrapper{}).
 		Complete(r)
 }
 
+func ocmSecretExists(namespace string) bool {
+	instascaleOCMSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "instascale-ocm-secret", metav1.GetOptions{})
+	if err != nil {
+		klog.Errorf("Error getting instascale-ocm-secret from namespace %v: %v", namespace, err)
+		klog.Infof("If you are looking to use OCM, ensure that the 'instascale-ocm-secret' secret is available on the cluster within namespace %v", namespace)
+		klog.Infof("Setting useMachineSets to %v.", useMachineSets)
+		return false
+	}
+
+	ocmToken = string(instascaleOCMSecret.Data["token"])
+	return true
+}
+
 func addAppwrappersThatNeedScaling() {
 	kubeconfig := os.Getenv("KUBECONFIG")
 	restConfig, err := getRestConfig(kubeconfig)
@@ -223,28 +237,27 @@ func onAdd(obj interface{}) {
 	aw, ok := obj.(*arbv1.AppWrapper)
 	if ok {
 		klog.Infof("Found Appwrapper named %s that has status %v", aw.Name, aw.Status.State)
-		if aw.Status.State == arbv1.AppWrapperStateEnqueued || aw.Status.State == "" {
+		if aw.Status.State == arbv1.AppWrapperStateEnqueued || aw.Status.State == "" && aw.Labels["orderedinstance"] != "" {
 			//scaledAppwrapper = append(scaledAppwrapper, aw.Name)
 			demandPerInstanceType := discoverInstanceTypes(aw)
 			//TODO: simplify the looping
-			if useMachineSets {
-				if canScaleMachineset(demandPerInstanceType) {
-					scaleUp(aw, demandPerInstanceType)
+			if demandPerInstanceType != nil {
+				if useMachineSets {
+					if canScaleMachineset(demandPerInstanceType) {
+						scaleUp(aw, demandPerInstanceType)
+					} else {
+						klog.Infof("Cannot scale up replicas max replicas allowed is %v", maxScaleNodesAllowed)
+					}
 				} else {
-					klog.Infof("Cannot scale up replicas max replicas allowed is %v", maxScaleNodesAllowed)
-				}
-			} else {
-				if canScaleMachinepool(demandPerInstanceType) {
-					scaleUp(aw, demandPerInstanceType)
-				} else {
-					klog.Infof("Cannot scale up replicas max replicas allowed is %v", maxScaleNodesAllowed)
+					if canScaleMachinepool(demandPerInstanceType) {
+						scaleUp(aw, demandPerInstanceType)
+					} else {
+						klog.Infof("Cannot scale up replicas max replicas allowed is %v", maxScaleNodesAllowed)
+					}
 				}
 			}
-
 		}
-
 	}
-
 }
 
 func onUpdate(old, new interface{}) {

controllers/machinepools.go

@@ -86,6 +86,30 @@ func deleteMachinePool(aw *arbv1.AppWrapper) {
 	})
 }
 
+// Check if machine pools exist
+func machinePoolExists() bool {
+	logger, err := ocmsdk.NewGoLoggerBuilder().
+		Debug(false).
+		Build()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Can't build logger: %v\n", err)
+		os.Exit(1)
+	}
+	connection, err := ocmsdk.NewConnectionBuilder().
+		Logger(logger).
+		Tokens(ocmToken).
+		Build()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err)
+		os.Exit(1)
+	}
+	defer connection.Close()
+
+	machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ocmClusterID).MachinePools()
+	klog.Infof("Machine pools: %v", machinePools)
+	return machinePools != nil
+}
+
 // getOCMClusterID determines the internal clusterID to be used for OCM API calls
 func getOCMClusterID(r *AppWrapperReconciler) error {
 

deployment/instascale-clusterrole.yaml

@@ -67,3 +67,4 @@ rules:
   verbs:
     - get
     - list
+    - watch

main.go

@@ -55,9 +55,11 @@ func main() {
 	var enableLeaderElection bool
 	var probeAddr string
 	var configsNamespace string
+	var ocmSecretNamespace string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.StringVar(&configsNamespace, "configs-namespace", "kube-system", "The namespace containing the InstaScale configmap and OCM secret")
+	flag.StringVar(&configsNamespace, "configs-namespace", "kube-system", "The namespace containing the Instacale configmap")
+	flag.StringVar(&ocmSecretNamespace, "ocm-secret-namespace", "default", "The namespace containing the OCM secret")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -83,9 +85,10 @@ func main() {
 	}
 
 	if err = (&controllers.AppWrapperReconciler{
-		Client:           mgr.GetClient(),
-		Scheme:           mgr.GetScheme(),
-		ConfigsNamespace: configsNamespace,
+		Client:             mgr.GetClient(),
+		Scheme:             mgr.GetScheme(),
+		ConfigsNamespace:   configsNamespace,
+		OcmSecretNamespace: ocmSecretNamespace,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "AppWrapper")
 		os.Exit(1)