From 28c96f962ce546ea78699ebd7a790f953fcd0e0a Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 03:59:09 +0530 Subject: [PATCH 01/10] add provider spec --- main.go | 6 +-- src/api/v1alpha1/uffizzicluster_types.go | 5 ++- .../uffizzicluster_controller.go | 3 +- src/pkg/constants/constants.go | 12 +++--- src/pkg/helm/build/vcluster/build.go | 39 ++++++++++++------- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/main.go b/main.go index d520b73b..071ca06d 100644 --- a/main.go +++ b/main.go @@ -72,7 +72,6 @@ func main() { "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.IntVar(&concurrentReconciliations, "concurrent", 5, "The number of concurrent reconciles per controller.") - flag.StringVar(&k8sProvider, "k8s-provider", "", "The k8s provider to use for the UffizziCluster") opts := zap.Options{ Development: true, } @@ -101,9 +100,8 @@ func main() { } // Setup UffizziClusterReconciler if err = (&uffizzicluster.UffizziClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - KubernetesProvider: k8sProvider, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "UffizziCluster") os.Exit(1) diff --git a/src/api/v1alpha1/uffizzicluster_types.go b/src/api/v1alpha1/uffizzicluster_types.go index 7151475a..70d094bc 100644 --- a/src/api/v1alpha1/uffizzicluster_types.go +++ b/src/api/v1alpha1/uffizzicluster_types.go @@ -143,7 +143,10 @@ type UffizziClusterResourceCount struct { type UffizziClusterSpec struct { //+kubebuilder:default:="k3s" //+kubebuilder:validation:Enum=k3s;k8s - Distro string `json:"distro,omitempty"` + Distro string `json:"distro,omitempty"` + //+kubebuilder:default:="vanila" + //+kubebuilder:validation:Enum=vanila,gke,eks + Provider string `json:"provider,omitempty"` APIServer UffizziClusterAPIServer `json:"apiServer,omitempty"` Ingress UffizziClusterIngress `json:"ingress,omitempty"` TTL string `json:"ttl,omitempty"` diff --git a/src/controllers/uffizzicluster/uffizzicluster_controller.go b/src/controllers/uffizzicluster/uffizzicluster_controller.go index a8e56e3d..2c12434d 100644 --- a/src/controllers/uffizzicluster/uffizzicluster_controller.go +++ b/src/controllers/uffizzicluster/uffizzicluster_controller.go @@ -42,8 +42,7 @@ import ( // UffizziClusterReconciler reconciles a UffizziCluster object type UffizziClusterReconciler struct { client.Client - Scheme *runtime.Scheme - KubernetesProvider string + Scheme *runtime.Scheme } //+kubebuilder:rbac:groups=uffizzi.com,resources=uffizziclusters,verbs=get;list;watch;create;update;patch;delete diff --git a/src/pkg/constants/constants.go b/src/pkg/constants/constants.go index 104a66ba..237d2d55 100644 --- a/src/pkg/constants/constants.go +++ b/src/pkg/constants/constants.go @@ -17,6 +17,8 @@ const ( LOFT_CHART_REPO_URL = "https://charts.loft.sh" VCLUSTER_K3S_DISTRO = "k3s" VCLUSTER_K8S_DISTRO = "k8s" + PROVIDER_GKE = "gke" + PROVIDER_EKS = "eks" K3S_DATASTORE_ENDPOINT = "K3S_DATASTORE_ENDPOINT" VCLUSTER_INGRESS_HOSTNAME = "VCLUSTER_INGRESS_HOST" DEFAULT_K3S_VERSION = "rancher/k3s:v1.27.3-k3s1" @@ -31,13 +33,9 @@ const ( ) type LIFECYCLE_OP_TYPE string -type KUBERNETES_PROVIDER string const ( - LIFECYCLE_OP_TYPE_CREATE LIFECYCLE_OP_TYPE = "create" - LIFECYCLE_OP_TYPE_UPDATE LIFECYCLE_OP_TYPE = "update" - LIFECYCLE_OP_TYPE_DELETE LIFECYCLE_OP_TYPE = "delete" - NO_KUBE_PROVIDER KUBERNETES_PROVIDER = "" - GKE_KUBE_PROVIDER KUBERNETES_PROVIDER = "gke" - EKS_KUBE_PROVIDER KUBERNETES_PROVIDER = "eks" + LIFECYCLE_OP_TYPE_CREATE LIFECYCLE_OP_TYPE = "create" + LIFECYCLE_OP_TYPE_UPDATE LIFECYCLE_OP_TYPE = "update" + LIFECYCLE_OP_TYPE_DELETE LIFECYCLE_OP_TYPE = "delete" ) diff --git a/src/pkg/helm/build/vcluster/build.go b/src/pkg/helm/build/vcluster/build.go index a16316ef..db7f9f1f 100644 --- a/src/pkg/helm/build/vcluster/build.go +++ b/src/pkg/helm/build/vcluster/build.go @@ -15,7 +15,7 @@ func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string vclusterK3sHelmValues := vcluster.K3S{ VCluster: k3SAPIServer(uCluster), - Common: common(helmReleaseName, vclusterIngressHostname), + Common: common(helmReleaseName, vclusterIngressHostname, uCluster.Spec.Provider), } // keep cluster data intact in case the vcluster scales up or down @@ -107,7 +107,7 @@ func BuildK8SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K8S, string vclusterHelmValues := vcluster.K8S{ APIServer: k8SAPIServer(), - Common: common(helmReleaseName, vclusterIngressHostname), + Common: common(helmReleaseName, vclusterIngressHostname, uCluster.Spec.Provider), } if uCluster.Spec.APIServer.Image != "" { @@ -213,10 +213,16 @@ func pluginsConfig() vcluster.Plugins { } } -func syncerConfig(helmReleaseName string) vcluster.Syncer { - return vcluster.Syncer{ +func syncerConfig(helmReleaseName, provider string) vcluster.Syncer { + syncer := vcluster.Syncer{ KubeConfigContextName: helmReleaseName, - ExtraArgs: []string{ + Limits: types.ContainerMemoryCPU{ + CPU: "1000m", + Memory: "1024Mi", + }, + } + if provider == constants.PROVIDER_GKE { + syncer.ExtraArgs = append(syncer.ExtraArgs, []string{ fmt.Sprintf( "--enforce-toleration=%s:%s", constants.SANDBOX_GKE_IO_RUNTIME, @@ -224,12 +230,10 @@ func syncerConfig(helmReleaseName string) vcluster.Syncer { ), "--node-selector=sandbox.gke.io/runtime=gvisor", "--enforce-node-selector", - }, - Limits: types.ContainerMemoryCPU{ - CPU: "1000m", - Memory: "1024Mi", - }, + }...) + } + return syncer } func syncConfig() vcluster.Sync { @@ -311,7 +315,7 @@ func ingress(VClusterIngressHostname string) vcluster.Ingress { } } -func nodeSelector() vcluster.NodeSelector { +func gkeNodeSelector() vcluster.NodeSelector { return vcluster.NodeSelector{ SandboxGKEIORuntime: "gvisor", } @@ -342,19 +346,24 @@ func k8SAPIServer() vcluster.K8SAPIServer { } } -func common(helmReleaseName, vclusterIngressHostname string) vcluster.Common { - return vcluster.Common{ +func common(helmReleaseName, vclusterIngressHostname, provider string) vcluster.Common { + c := vcluster.Common{ Init: vcluster.Init{}, FsGroup: 12345, Ingress: ingress(vclusterIngressHostname), Isolation: isolation(), - NodeSelector: nodeSelector(), SecurityContext: securityContext(), Tolerations: tolerations(), Plugin: pluginsConfig(), - Syncer: syncerConfig(helmReleaseName), + Syncer: syncerConfig(helmReleaseName, provider), Sync: syncConfig(), } + + if provider == constants.PROVIDER_GKE { + c.NodeSelector = gkeNodeSelector() + } + + return c } func k3SAPIServer(uCluster *v1alpha1.UffizziCluster) vcluster.K3SAPIServer { From 96197b53aa7139b3bab4ba1955f41ef8f3d4db07 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 04:03:42 +0530 Subject: [PATCH 02/10] delete random redundant package created by mistake --- controllers/constants/constants.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 controllers/constants/constants.go diff --git a/controllers/constants/constants.go b/controllers/constants/constants.go deleted file mode 100644 index 1255fc8b..00000000 --- a/controllers/constants/constants.go +++ /dev/null @@ -1 +0,0 @@ -package constants From cb06bb0a8ad8b0baa2a50dd1236965c9c939f557 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 04:08:24 +0530 Subject: [PATCH 03/10] fix enum types --- src/api/v1alpha1/uffizzicluster_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/v1alpha1/uffizzicluster_types.go b/src/api/v1alpha1/uffizzicluster_types.go index 70d094bc..30200ab2 100644 --- a/src/api/v1alpha1/uffizzicluster_types.go +++ b/src/api/v1alpha1/uffizzicluster_types.go @@ -145,7 +145,7 @@ type UffizziClusterSpec struct { //+kubebuilder:validation:Enum=k3s;k8s Distro string `json:"distro,omitempty"` //+kubebuilder:default:="vanila" - //+kubebuilder:validation:Enum=vanila,gke,eks + //+kubebuilder:validation:Enum=vanila;gke;eks Provider string `json:"provider,omitempty"` APIServer UffizziClusterAPIServer `json:"apiServer,omitempty"` Ingress UffizziClusterIngress `json:"ingress,omitempty"` From 479b385bd5d99449a603466d2054babeaa85ab1b Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 04:11:04 +0530 Subject: [PATCH 04/10] unused var --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 071ca06d..d1cd083c 100644 --- a/main.go +++ b/main.go @@ -63,7 +63,6 @@ func main() { probeAddr string enableLeaderElection bool concurrentReconciliations int - k8sProvider string ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") From 5d98a1df0c86934bd19f08e1a072a18aa6a55e1a Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 04:23:38 +0530 Subject: [PATCH 05/10] update config --- config/crd/bases/uffizzi.com_uffizziclusters.yaml | 7 +++++++ src/test/e2e/uffizzicluster_controller_test.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/config/crd/bases/uffizzi.com_uffizziclusters.yaml b/config/crd/bases/uffizzi.com_uffizziclusters.yaml index 0df7493c..500cc20b 100644 --- a/config/crd/bases/uffizzi.com_uffizziclusters.yaml +++ b/config/crd/bases/uffizzi.com_uffizziclusters.yaml @@ -154,6 +154,13 @@ spec: type: object manifests: type: string + provider: + default: vanila + enum: + - vanila + - gke + - eks + type: string resourceQuota: description: UffizziClusterResourceQuota defines the resource quota which defines the quota of resources a namespace has access to diff --git a/src/test/e2e/uffizzicluster_controller_test.go b/src/test/e2e/uffizzicluster_controller_test.go index 55267dfb..0b61fbdf 100644 --- a/src/test/e2e/uffizzicluster_controller_test.go +++ b/src/test/e2e/uffizzicluster_controller_test.go @@ -93,6 +93,21 @@ var _ = Describe("UffizziCluster Controller", func() { d := cmp.Diff(expectedConditions, uc.Status.Conditions) GinkgoWriter.Printf(diff.PrintWantGot(d)) }) + + It("Should be in a Ready State", func() { + expectedConditions := []metav1.Condition{} + uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) + By("Check if UffizziCluster has the correct Ready conditions") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + expectedConditions = uffizzicluster.GetAllReadyConditions() + return containsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) }) }) From 2bdec7976e1de99f690546d35f7088404cbd1df0 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 13:41:50 +0530 Subject: [PATCH 06/10] restore sleep state reconcile to enable the ready state e2e --- chart/templates/manager-role_clusterrole.yaml | 4 +--- .../uffizzicluster_controller.go | 19 ++++++++------- src/pkg/constants/constants.go | 1 + src/pkg/helm/build/vcluster/build.go | 11 ++++++--- src/pkg/helm/types/vcluster/vcluster.go | 2 +- .../e2e/uffizzicluster_controller_test.go | 24 +++++++++++++++++++ 6 files changed, 45 insertions(+), 16 deletions(-) diff --git a/chart/templates/manager-role_clusterrole.yaml b/chart/templates/manager-role_clusterrole.yaml index 55661338..a6b38a51 100644 --- a/chart/templates/manager-role_clusterrole.yaml +++ b/chart/templates/manager-role_clusterrole.yaml @@ -2,9 +2,7 @@ apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} kind: ClusterRole metadata: - labels: {{ include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: rbac - app.kubernetes.io/part-of: uffizzi +labels: {{ include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: rbac app.kubernetes.io/part-of: uffizzi name: {{ include "common.names.fullname" . }}-manager-role rules: - apiGroups: diff --git a/src/controllers/uffizzicluster/uffizzicluster_controller.go b/src/controllers/uffizzicluster/uffizzicluster_controller.go index 2c12434d..cc5f7e16 100644 --- a/src/controllers/uffizzicluster/uffizzicluster_controller.go +++ b/src/controllers/uffizzicluster/uffizzicluster_controller.go @@ -249,15 +249,15 @@ func (r *UffizziClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque // ---------------------- // UCLUSTER SLEEP // ---------------------- - //if err := r.reconcileSleepState(ctx, uCluster); err != nil { - // if k8serrors.IsNotFound(err) { - // // logger.Info("vcluster statefulset not found, requeueing") - // return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil - // } - // // cluster did not sleep - // logger.Info("Failed to reconcile sleep state, reconciling again", "Error", err.Error()) - // return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil - //} + if err := r.reconcileSleepState(ctx, uCluster); err != nil { + if k8serrors.IsNotFound(err) { + logger.Info("vcluster statefulset not found, will check again in the next round") + return ctrl.Result{}, nil + } + // cluster did not sleep + logger.Info("Failed to reconcile sleep state, reconciling again", "Error", err.Error()) + return ctrl.Result{}, nil + } return ctrl.Result{}, nil } @@ -302,6 +302,7 @@ func (r *UffizziClusterReconciler) reconcileSleepState(ctx context.Context, uClu // patch := client.MergeFrom(uCluster.DeepCopy()) // get the stateful set or deployment created by the helm chart + // k3s is a statefulset, k8s is a helmchart ucWorkload, err := r.getUffizziClusterWorkload(ctx, uCluster) if err != nil { return err diff --git a/src/pkg/constants/constants.go b/src/pkg/constants/constants.go index 237d2d55..a2de30c8 100644 --- a/src/pkg/constants/constants.go +++ b/src/pkg/constants/constants.go @@ -17,6 +17,7 @@ const ( LOFT_CHART_REPO_URL = "https://charts.loft.sh" VCLUSTER_K3S_DISTRO = "k3s" VCLUSTER_K8S_DISTRO = "k8s" + PROVIDER_NOTHING = "vanila" PROVIDER_GKE = "gke" PROVIDER_EKS = "eks" K3S_DATASTORE_ENDPOINT = "K3S_DATASTORE_ENDPOINT" diff --git a/src/pkg/helm/build/vcluster/build.go b/src/pkg/helm/build/vcluster/build.go index db7f9f1f..3a7d2637 100644 --- a/src/pkg/helm/build/vcluster/build.go +++ b/src/pkg/helm/build/vcluster/build.go @@ -231,7 +231,8 @@ func syncerConfig(helmReleaseName, provider string) vcluster.Syncer { "--node-selector=sandbox.gke.io/runtime=gvisor", "--enforce-node-selector", }...) - + } else { + syncer.ExtraArgs = []string{} } return syncer } @@ -244,7 +245,7 @@ func syncConfig() vcluster.Sync { } } -func tolerations() []vcluster.Toleration { +func gkeTolerations() []vcluster.Toleration { return []vcluster.Toleration{ { Key: constants.SANDBOX_GKE_IO_RUNTIME, @@ -353,7 +354,7 @@ func common(helmReleaseName, vclusterIngressHostname, provider string) vcluster. Ingress: ingress(vclusterIngressHostname), Isolation: isolation(), SecurityContext: securityContext(), - Tolerations: tolerations(), + Tolerations: gkeTolerations(), Plugin: pluginsConfig(), Syncer: syncerConfig(helmReleaseName, provider), Sync: syncConfig(), @@ -361,6 +362,10 @@ func common(helmReleaseName, vclusterIngressHostname, provider string) vcluster. if provider == constants.PROVIDER_GKE { c.NodeSelector = gkeNodeSelector() + c.Tolerations = gkeTolerations() + } else { + c.NodeSelector = vcluster.NodeSelector{} + c.Tolerations = []vcluster.Toleration{} } return c diff --git a/src/pkg/helm/types/vcluster/vcluster.go b/src/pkg/helm/types/vcluster/vcluster.go index 6d9542ec..cb2fdfd3 100644 --- a/src/pkg/helm/types/vcluster/vcluster.go +++ b/src/pkg/helm/types/vcluster/vcluster.go @@ -190,7 +190,7 @@ type Isolation struct { // NodeSelector - parameters to define the node selector of the cluster type NodeSelector struct { - SandboxGKEIORuntime string `json:"sandbox.gke.io/runtime"` + SandboxGKEIORuntime string `json:"sandbox.gke.io/runtime,omitempty"` } type SecurityContextCapabilities struct { diff --git a/src/test/e2e/uffizzicluster_controller_test.go b/src/test/e2e/uffizzicluster_controller_test.go index 0b61fbdf..86ad2ffc 100644 --- a/src/test/e2e/uffizzicluster_controller_test.go +++ b/src/test/e2e/uffizzicluster_controller_test.go @@ -109,6 +109,30 @@ var _ = Describe("UffizziCluster Controller", func() { GinkgoWriter.Printf(diff.PrintWantGot(d)) }) }) + + //Context("When putting a cluster to sleep", func() { + // It("Should put the cluster to sleep", func() { + // By("By putting the UffizziCluster to sleep") + // uc.Spec.Sleep = true + // Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) + // }) + // + // It("Should be in a Sleep State", func() { + // expectedConditions := []metav1.Condition{ + // uffizzicluster.Sleeping(metav1.Now()), + // } + // uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) + // By("Check if UffizziCluster has the correct Sleep conditions") + // Eventually(func() bool { + // if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + // return false + // } + // return containsAllConditions(expectedConditions, uc.Status.Conditions) + // }, timeout, pollingTimeout).Should(BeTrue()) + // d := cmp.Diff(expectedConditions, uc.Status.Conditions) + // GinkgoWriter.Printf(diff.PrintWantGot(d)) + // }) + //}) }) func createNamespacesName(name, namespace string) types.NamespacedName { From ae858d18a5a7a7a6b87e93cb45dc832a253acfb4 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 27 Feb 2024 13:44:10 +0530 Subject: [PATCH 07/10] fix lint --- chart/templates/manager-role_clusterrole.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chart/templates/manager-role_clusterrole.yaml b/chart/templates/manager-role_clusterrole.yaml index a6b38a51..55661338 100644 --- a/chart/templates/manager-role_clusterrole.yaml +++ b/chart/templates/manager-role_clusterrole.yaml @@ -2,7 +2,9 @@ apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} kind: ClusterRole metadata: -labels: {{ include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: rbac app.kubernetes.io/part-of: uffizzi + labels: {{ include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: rbac + app.kubernetes.io/part-of: uffizzi name: {{ include "common.names.fullname" . }}-manager-role rules: - apiGroups: From bca6ac4c3100cfea52c42780a6f95f3378571f5e Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Wed, 28 Feb 2024 01:46:34 +0530 Subject: [PATCH 08/10] fix sleep scaling --- .../uffizzicluster_controller.go | 6 +++ .../e2e/uffizzicluster_controller_test.go | 46 +++++++++---------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/controllers/uffizzicluster/uffizzicluster_controller.go b/src/controllers/uffizzicluster/uffizzicluster_controller.go index cc5f7e16..40cecd12 100644 --- a/src/controllers/uffizzicluster/uffizzicluster_controller.go +++ b/src/controllers/uffizzicluster/uffizzicluster_controller.go @@ -319,10 +319,16 @@ func (r *UffizziClusterReconciler) reconcileSleepState(ctx context.Context, uClu // scale the vcluster instance to 0 if the sleep flag is true if uCluster.Spec.Sleep && *currentReplicas > 0 { var err error + if err = r.scaleStatefulSets(ctx, 0, ucStatefulSet); err != nil { + return err + } if err = r.waitForStatefulSetToScale(ctx, 0, ucStatefulSet); err == nil { setCondition(uCluster, APINotReady()) } if uCluster.Spec.ExternalDatastore == constants.ETCD { + if err = r.scaleStatefulSets(ctx, 0, etcdStatefulSet); err != nil { + return err + } if err = r.waitForStatefulSetToScale(ctx, 0, etcdStatefulSet); err == nil { setCondition(uCluster, DataStoreNotReady()) } diff --git a/src/test/e2e/uffizzicluster_controller_test.go b/src/test/e2e/uffizzicluster_controller_test.go index 86ad2ffc..07ef12ce 100644 --- a/src/test/e2e/uffizzicluster_controller_test.go +++ b/src/test/e2e/uffizzicluster_controller_test.go @@ -110,29 +110,29 @@ var _ = Describe("UffizziCluster Controller", func() { }) }) - //Context("When putting a cluster to sleep", func() { - // It("Should put the cluster to sleep", func() { - // By("By putting the UffizziCluster to sleep") - // uc.Spec.Sleep = true - // Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) - // }) - // - // It("Should be in a Sleep State", func() { - // expectedConditions := []metav1.Condition{ - // uffizzicluster.Sleeping(metav1.Now()), - // } - // uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) - // By("Check if UffizziCluster has the correct Sleep conditions") - // Eventually(func() bool { - // if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { - // return false - // } - // return containsAllConditions(expectedConditions, uc.Status.Conditions) - // }, timeout, pollingTimeout).Should(BeTrue()) - // d := cmp.Diff(expectedConditions, uc.Status.Conditions) - // GinkgoWriter.Printf(diff.PrintWantGot(d)) - // }) - //}) + Context("When putting a cluster to sleep", func() { + It("Should put the cluster to sleep", func() { + By("By putting the UffizziCluster to sleep") + uc.Spec.Sleep = true + Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) + }) + + It("Should be in a Sleep State", func() { + expectedConditions := []metav1.Condition{ + uffizzicluster.APINotReady(), + } + uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) + By("Check if UffizziCluster has the correct Sleep conditions") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + return containsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) + }) }) func createNamespacesName(name, namespace string) types.NamespacedName { From 8e9ea773d18de61b7c45c2de03dd1b67602a981c Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Wed, 28 Feb 2024 02:16:05 +0530 Subject: [PATCH 09/10] refactor sleep state reconciliation --- .../uffizzicluster_controller.go | 181 +++++++----------- src/controllers/uffizzicluster/workload.go | 62 ++++++ .../e2e/uffizzicluster_controller_test.go | 37 ++++ 3 files changed, 166 insertions(+), 114 deletions(-) diff --git a/src/controllers/uffizzicluster/uffizzicluster_controller.go b/src/controllers/uffizzicluster/uffizzicluster_controller.go index 40cecd12..b437b9ce 100644 --- a/src/controllers/uffizzicluster/uffizzicluster_controller.go +++ b/src/controllers/uffizzicluster/uffizzicluster_controller.go @@ -19,6 +19,7 @@ package uffizzicluster import ( "context" "encoding/json" + "fmt" "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/helm/build/vcluster" @@ -298,135 +299,87 @@ func (r *UffizziClusterReconciler) createEgressPolicy(ctx context.Context, uClus // reconcileSleepState reconciles the sleep state of the vcluster // it also makes sure that the vcluster is up and running before setting the sleep state func (r *UffizziClusterReconciler) reconcileSleepState(ctx context.Context, uCluster *v1alpha1.UffizziCluster) error { - // get the patch copy of the uCluster so that we can have a good diff between the uCluster and patch object - // - patch := client.MergeFrom(uCluster.DeepCopy()) + var ( + // get the patch copy of the uCluster so that we can have a good diff between the uCluster and patch object + patch = client.MergeFrom(uCluster.DeepCopy()) + ucWorkload runtime.Object + etcdStatefulSet *appsv1.StatefulSet + err error + ) // get the stateful set or deployment created by the helm chart - // k3s is a statefulset, k8s is a helmchart - ucWorkload, err := r.getUffizziClusterWorkload(ctx, uCluster) - if err != nil { - return err + // k3s is a statefulset, k8s is a deployment + if ucWorkload, err = r.getUffizziClusterWorkload(ctx, uCluster); err != nil { + return fmt.Errorf("failed to get uffizzicluster workload: %w", err) } // get the etcd stateful set created by the helm chart - etcdStatefulSet, err := r.getEtcdStatefulSet(ctx, uCluster) + if uCluster.Spec.ExternalDatastore == constants.ETCD { + if etcdStatefulSet, err = r.getEtcdStatefulSet(ctx, uCluster); err != nil { + return fmt.Errorf("failed to get etcd statefulset: %w", err) + } + } // execute sleep reconciliation based on the type of workload - // TODO: Abstract the actual sleep reconciliation logic into a separate function so that it can be reused - // for different types of workloads, i.e. statefulset, deployment, daemonset - switch ucWorkload.(type) { - case *appsv1.StatefulSet: - ucStatefulSet := ucWorkload.(*appsv1.StatefulSet) - currentReplicas := ucStatefulSet.Spec.Replicas - // scale the vcluster instance to 0 if the sleep flag is true - if uCluster.Spec.Sleep && *currentReplicas > 0 { - var err error - if err = r.scaleStatefulSets(ctx, 0, ucStatefulSet); err != nil { + currentReplicas := r.getReplicasForWorkload(ucWorkload) + // scale the vcluster instance to 0 if the sleep flag is true + if uCluster.Spec.Sleep && currentReplicas > 0 { + if err = r.scaleWorkloads(ctx, 0, ucWorkload); err != nil { + return err + } + if err = r.waitForWorkloadToScale(ctx, 0, ucWorkload); err == nil { + setCondition(uCluster, APINotReady()) + } + if uCluster.Spec.ExternalDatastore == constants.ETCD { + if err = r.scaleWorkloads(ctx, 0, etcdStatefulSet); err != nil { return err } - if err = r.waitForStatefulSetToScale(ctx, 0, ucStatefulSet); err == nil { - setCondition(uCluster, APINotReady()) - } - if uCluster.Spec.ExternalDatastore == constants.ETCD { - if err = r.scaleStatefulSets(ctx, 0, etcdStatefulSet); err != nil { - return err - } - if err = r.waitForStatefulSetToScale(ctx, 0, etcdStatefulSet); err == nil { - setCondition(uCluster, DataStoreNotReady()) - } - } else { + if err = r.waitForWorkloadToScale(ctx, 0, etcdStatefulSet); err == nil { setCondition(uCluster, DataStoreNotReady()) } - if err != nil { - return err - } - err = r.deleteWorkloads(ctx, uCluster) - if err != nil { - return err - } - sleepingTime := metav1.Now().Rfc3339Copy() - setCondition(uCluster, Sleeping(sleepingTime)) - // if the current replicas is 0, then do nothing - } else if !uCluster.Spec.Sleep && *currentReplicas == 0 { - statefulSets := []*appsv1.StatefulSet{} - statefulSets = append(statefulSets, ucStatefulSet) - if uCluster.Spec.ExternalDatastore == constants.ETCD { - statefulSets = append(statefulSets, etcdStatefulSet) - } - if err := r.scaleStatefulSets(ctx, 1, statefulSets...); err != nil { - return err - } + } else { + setCondition(uCluster, DataStoreNotReady()) } - // ensure that the statefulset is up if the cluster is not sleeping - if !uCluster.Spec.Sleep { - // set status for vcluster waking up - lastAwakeTime := metav1.Now().Rfc3339Copy() - uCluster.Status.LastAwakeTime = lastAwakeTime - // if the above runs successfully, then set the status to awake - setCondition(uCluster, Awoken(lastAwakeTime)) - var err error - if uCluster.Spec.ExternalDatastore == constants.ETCD { - if err = r.waitForStatefulSetToScale(ctx, 1, etcdStatefulSet); err == nil { - setCondition(uCluster, DataStoreReady()) - } - } else { - setCondition(uCluster, DataStoreReady()) - } - if err = r.waitForStatefulSetToScale(ctx, 1, ucStatefulSet); err == nil { - setCondition(uCluster, APIReady()) - } - if err != nil { - return err - } + if err != nil { + return err } - case *appsv1.Deployment: - ucDeployment := ucWorkload.(*appsv1.Deployment) - currentReplicas := ucDeployment.Spec.Replicas - // scale the vcluster instance to 0 if the sleep flag is true - if uCluster.Spec.Sleep && *currentReplicas > 0 { - var err error - if err = r.waitForDeploymentToScale(ctx, 0, ucDeployment); err == nil { - setCondition(uCluster, APINotReady()) - } - if err = r.waitForStatefulSetToScale(ctx, 0, etcdStatefulSet); err == nil { - setCondition(uCluster, DataStoreNotReady()) - } - if err != nil { - return err - } - - err = r.deleteWorkloads(ctx, uCluster) - if err != nil { - return err - } - sleepingTime := metav1.Now().Rfc3339Copy() - setCondition(uCluster, Sleeping(sleepingTime)) - // if the current replicas is 0, then do nothing - } else if !uCluster.Spec.Sleep && *currentReplicas == 0 { - if err := r.scaleDeployments(ctx, 1, ucDeployment); err != nil { - return err - } + err = r.deleteWorkloads(ctx, uCluster) + if err != nil { + return err } - // ensure that the deployment is up if the cluster is not sleeping - if !uCluster.Spec.Sleep { - // set status for vcluster waking up - lastAwakeTime := metav1.Now().Rfc3339Copy() - uCluster.Status.LastAwakeTime = lastAwakeTime - // if the above runs successfully, then set the status to awake - setCondition(uCluster, Awoken(lastAwakeTime)) - var err error - if err = r.waitForStatefulSetToScale(ctx, 1, etcdStatefulSet); err == nil { - setCondition(uCluster, APIReady()) - } - if err = r.waitForDeploymentToScale(ctx, 1, ucDeployment); err == nil { + sleepingTime := metav1.Now().Rfc3339Copy() + setCondition(uCluster, Sleeping(sleepingTime)) + // if the current replicas is 0, then do nothing + } else if !uCluster.Spec.Sleep && currentReplicas == 0 { + workloads := []runtime.Object{} + workloads = append(workloads, ucWorkload) + if uCluster.Spec.ExternalDatastore == constants.ETCD { + workloads = append(workloads, etcdStatefulSet) + } + if err := r.scaleWorkloads(ctx, 1, workloads...); err != nil { + return err + } + } + // ensure that the statefulset is up if the cluster is not sleeping + if !uCluster.Spec.Sleep { + // set status for vcluster waking up + lastAwakeTime := metav1.Now().Rfc3339Copy() + uCluster.Status.LastAwakeTime = lastAwakeTime + // if the above runs successfully, then set the status to awake + setCondition(uCluster, Awoken(lastAwakeTime)) + var err error + if uCluster.Spec.ExternalDatastore == constants.ETCD { + if err = r.waitForWorkloadToScale(ctx, 1, etcdStatefulSet); err == nil { setCondition(uCluster, DataStoreReady()) } - if err != nil { - return err - } + } else { + setCondition(uCluster, DataStoreReady()) + } + if err = r.waitForWorkloadToScale(ctx, 1, ucWorkload); err == nil { + setCondition(uCluster, APIReady()) + } + if err != nil { + return err } - - default: - return errors.New("unknown workload type for vcluster") } + if err := r.Status().Patch(ctx, uCluster, patch); err != nil { return err } diff --git a/src/controllers/uffizzicluster/workload.go b/src/controllers/uffizzicluster/workload.go index 0ea7118d..fa12647a 100644 --- a/src/controllers/uffizzicluster/workload.go +++ b/src/controllers/uffizzicluster/workload.go @@ -2,9 +2,11 @@ package uffizzicluster import ( context "context" + "fmt" "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/helm/build/vcluster" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,3 +35,63 @@ func (r *UffizziClusterReconciler) deleteWorkloads(ctx context.Context, uc *v1al } return nil } + +func (r *UffizziClusterReconciler) scaleWorkloads(ctx context.Context, scale int, workloads ...runtime.Object) error { + sss := []*appsv1.StatefulSet{} + ds := []*appsv1.Deployment{} + for _, w := range workloads { + if w != nil { + switch w.(type) { + case *appsv1.StatefulSet: + ss := w.(*appsv1.StatefulSet) + sss = append(sss, ss) + case *appsv1.Deployment: + d := w.(*appsv1.Deployment) + ds = append(ds, d) + } + } + } + if len(sss) > 0 { + if err := r.scaleStatefulSets(ctx, scale, sss...); err != nil { + return fmt.Errorf("failed to scale stateful sets: %w", err) + } + } + if len(ds) > 0 { + if err := r.scaleDeployments(ctx, scale, ds...); err != nil { + return fmt.Errorf("failed to scale deployments: %w", err) + } + } + return nil +} + +func (r *UffizziClusterReconciler) waitForWorkloadToScale(ctx context.Context, scale int, w runtime.Object) error { + if w != nil { + switch w.(type) { + case *appsv1.StatefulSet: + ss := w.(*appsv1.StatefulSet) + if err := r.waitForStatefulSetToScale(ctx, scale, ss); err != nil { + return fmt.Errorf("failed to wait for stateful sets to scale: %w", err) + } + case *appsv1.Deployment: + d := w.(*appsv1.Deployment) + if err := r.waitForDeploymentToScale(ctx, scale, d); err != nil { + return fmt.Errorf("failed to wait for deployments to scale: %w", err) + } + } + } + return nil +} + +func (r *UffizziClusterReconciler) getReplicasForWorkload(w runtime.Object) int { + if w != nil { + switch w.(type) { + case *appsv1.StatefulSet: + ss := w.(*appsv1.StatefulSet) + return int(*ss.Spec.Replicas) + case *appsv1.Deployment: + d := w.(*appsv1.Deployment) + return int(*d.Spec.Replicas) + } + } + return 0 +} diff --git a/src/test/e2e/uffizzicluster_controller_test.go b/src/test/e2e/uffizzicluster_controller_test.go index 07ef12ce..4ba0d5d7 100644 --- a/src/test/e2e/uffizzicluster_controller_test.go +++ b/src/test/e2e/uffizzicluster_controller_test.go @@ -120,6 +120,7 @@ var _ = Describe("UffizziCluster Controller", func() { It("Should be in a Sleep State", func() { expectedConditions := []metav1.Condition{ uffizzicluster.APINotReady(), + uffizzicluster.DataStoreNotReady(), } uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) By("Check if UffizziCluster has the correct Sleep conditions") @@ -133,6 +134,42 @@ var _ = Describe("UffizziCluster Controller", func() { GinkgoWriter.Printf(diff.PrintWantGot(d)) }) }) + + Context("When waking a cluster up", func() { + It("Should wake the cluster up", func() { + By("By waking the UffizziCluster up") + uc.Spec.Sleep = false + Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) + }) + + It("Should be Awoken", func() { + expectedConditions := []metav1.Condition{ + uffizzicluster.APIReady(), + uffizzicluster.DataStoreReady(), + } + uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) + By("Check if UffizziCluster has the correct Sleep conditions") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + return containsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) + }) + + Context("When deleting UffizziCluster", func() { + It("Should delete the UffizziCluster", func() { + By("By deleting the UffizziCluster") + Expect(k8sClient.Delete(ctx, uc)).Should(Succeed()) + }) + It("Should delete the Namespace", func() { + By("By deleting the Namespace") + Expect(deleteTestNamespace(ns.Name)).Should(Succeed()) + }) + }) }) func createNamespacesName(name, namespace string) types.NamespacedName { From c7ba666db1b3dcca947db4f3659bd0fe6fa3cb40 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Wed, 28 Feb 2024 03:14:58 +0530 Subject: [PATCH 10/10] refactor lifecycle test --- src/test/e2e/suite_test.go | 44 +--- src/test/e2e/ucluster_lifecycle_test.go | 151 +++++++++++++ .../e2e/uffizzicluster_controller_test.go | 198 ++---------------- src/test/util/common.go | 19 ++ src/test/util/conditions/conditions.go | 30 +++ src/test/util/resources/resources.go | 61 ++++++ 6 files changed, 285 insertions(+), 218 deletions(-) create mode 100644 src/test/e2e/ucluster_lifecycle_test.go create mode 100644 src/test/util/common.go create mode 100644 src/test/util/conditions/conditions.go create mode 100644 src/test/util/resources/resources.go diff --git a/src/test/e2e/suite_test.go b/src/test/e2e/suite_test.go index 6a8e5b4b..43c3ff6d 100644 --- a/src/test/e2e/suite_test.go +++ b/src/test/e2e/suite_test.go @@ -20,20 +20,15 @@ import ( "context" uffizziv1alpha1 "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" "github.com/UffizziCloud/uffizzi-cluster-operator/src/controllers/uffizzicluster" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/diff" fluxhelmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1" fluxsourcev1beta2 "github.com/fluxcd/source-controller/api/v1beta2" - "github.com/google/go-cmp/cmp" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "math/rand" "os" "path/filepath" ctrl "sigs.k8s.io/controller-runtime" "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -132,19 +127,6 @@ func testingSeed() { rand.Seed(12345) } -func stringWithCharset(length int, charset string) string { - seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) - b := make([]byte, length) - for i := range b { - b[i] = charset[seededRand.Intn(len(charset))] - } - return string(b) -} - -func randomString(length int) string { - return stringWithCharset(length, "abcdefghijklmnopqrstuvwxyz") -} - // CompareSlices compares two slices for equality. It returns true if both slices have the same elements in the same order. func compareSlices[T comparable](slice1, slice2 []T) bool { if len(slice1) != len(slice2) { @@ -159,25 +141,3 @@ func compareSlices[T comparable](slice1, slice2 []T) bool { return true } - -// Checks if the required conditions are present and match in the actual conditions slice. -// Both requiredConditions and actualConditions are slices of metav1.Condition. -func containsAllConditions(requiredConditions, actualConditions []metav1.Condition) bool { - for _, requiredCondition := range requiredConditions { - found := false - for _, actualCondition := range actualConditions { - if actualCondition.Type == requiredCondition.Type && - actualCondition.Status == requiredCondition.Status { - // Add more condition checks here if necessary (e.g., Reason, Message) - found = true - d := cmp.Diff(requiredConditions, actualConditions) - GinkgoWriter.Printf(diff.PrintWantGot(d)) - break - } - } - if !found { - return false - } - } - return true -} diff --git a/src/test/e2e/ucluster_lifecycle_test.go b/src/test/e2e/ucluster_lifecycle_test.go new file mode 100644 index 00000000..a838aadf --- /dev/null +++ b/src/test/e2e/ucluster_lifecycle_test.go @@ -0,0 +1,151 @@ +package e2e + +import ( + context "context" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/controllers/uffizzicluster" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/conditions" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/diff" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/resources" + "github.com/google/go-cmp/cmp" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func wrapUffizziClusterLifecycleTest(ctx context.Context, ns *v1.Namespace, uc *v1alpha1.UffizziCluster) { + helmRelease := resources.GetHelmReleaseFromUffizziCluster(uc) + helmRepo := resources.GetHelmRepositoryFromUffizziCluster(uc) + defer Context("When deleting UffizziCluster", func() { + It("Should delete the UffizziCluster", func() { + By("By deleting the UffizziCluster") + Expect(k8sClient.Delete(ctx, uc)).Should(Succeed()) + }) + It("Should delete the Namespace", func() { + By("By deleting the Namespace") + Expect(deleteTestNamespace(ns.Name)).Should(Succeed()) + }) + }) + + Context("When creating UffizziCluster", func() { + It("Should create a UffizziCluster", func() { + // + By("By Creating Namespace for the UffizziCluster") + Expect(k8sClient.Create(ctx, ns)).Should(Succeed()) + + // + By("By creating a new UffizziCluster") + Expect(k8sClient.Create(ctx, uc)).Should(Succeed()) + }) + + It("Should create a HelmRelease and HelmRepository", func() { + // + By("Checking if the HelmRelease was created") + Eventually(func() bool { + if err := k8sClient.Get(ctx, resources.CreateNamespacedName(helmRelease.Name, ns.Name), helmRelease); err != nil { + return false + } + return true + }) + // + By("Checking if the Loft HelmRepository was created") + Eventually(func() bool { + if err := k8sClient.Get(ctx, resources.CreateNamespacedName(constants.LOFT_HELM_REPO, ns.Name), helmRepo); err != nil { + return false + } + return true + }) + + }) + + It("Should initialize correctly", func() { + expectedConditions := []metav1.Condition{} + uffizziClusterNSN := resources.CreateNamespacedName(uc.Name, ns.Name) + By("Check if UffizziCluster initializes correctly") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + expectedConditions = uffizzicluster.GetAllInitializingConditions() + return conditions.ContainsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) + + It("Should be in a Ready State", func() { + expectedConditions := []metav1.Condition{} + uffizziClusterNSN := resources.CreateNamespacedName(uc.Name, ns.Name) + By("Check if UffizziCluster has the correct Ready conditions") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + expectedConditions = uffizzicluster.GetAllReadyConditions() + return conditions.ContainsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) + }) + + Context("When putting a cluster to sleep", func() { + It("Should put the cluster to sleep", func() { + By("By putting the UffizziCluster to sleep") + uc.Spec.Sleep = true + Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) + }) + + It("Should be in a Sleep State", func() { + expectedConditions := []metav1.Condition{ + uffizzicluster.APINotReady(), + uffizzicluster.DataStoreNotReady(), + } + uffizziClusterNSN := resources.CreateNamespacedName(uc.Name, ns.Name) + By("Check if UffizziCluster has the correct Sleep conditions") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + return conditions.ContainsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) + }) + + Context("When waking a cluster up", func() { + It("Should wake the cluster up", func() { + By("By waking the UffizziCluster up") + uc.Spec.Sleep = false + Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) + }) + + It("Should be Awoken", func() { + expectedConditions := []metav1.Condition{ + uffizzicluster.APIReady(), + uffizzicluster.DataStoreReady(), + } + uffizziClusterNSN := resources.CreateNamespacedName(uc.Name, ns.Name) + By("Check if UffizziCluster has the correct Sleep conditions") + Eventually(func() bool { + if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { + return false + } + return conditions.ContainsAllConditions(expectedConditions, uc.Status.Conditions) + }, timeout, pollingTimeout).Should(BeTrue()) + d := cmp.Diff(expectedConditions, uc.Status.Conditions) + GinkgoWriter.Printf(diff.PrintWantGot(d)) + }) + }) +} + +func deleteTestNamespace(name string) error { + return k8sClient.Delete(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }) +} diff --git a/src/test/e2e/uffizzicluster_controller_test.go b/src/test/e2e/uffizzicluster_controller_test.go index 4ba0d5d7..b8ce1ef5 100644 --- a/src/test/e2e/uffizzicluster_controller_test.go +++ b/src/test/e2e/uffizzicluster_controller_test.go @@ -3,186 +3,32 @@ package e2e import ( "context" "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/controllers/uffizzicluster" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" - "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/diff" - fluxhelmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1" - fluxsourcev1beta2 "github.com/fluxcd/source-controller/api/v1beta2" - "github.com/google/go-cmp/cmp" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/resources" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "strings" ) -var _ = Describe("UffizziCluster Controller", func() { - ctx := context.Background() - testingSeed() - name := "basic" - timeout := "1m" - pollingTimeout := "100ms" - - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: strings.Join([]string{name, randomString(5)}, "-"), - }, - } - uc := &v1alpha1.UffizziCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns.Name, - }, - } - helmRelease := &fluxhelmv2beta1.HelmRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "uc-" + name, - Namespace: ns.Name, - }, - } - helmRepo := &fluxsourcev1beta2.HelmRepository{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.LOFT_HELM_REPO, - Namespace: ns.Name, - }, - } - - Context("When creating UffizziCluster", func() { - It("Should create a UffizziCluster", func() { - // - By("By Creating Namespace for the UffizziCluster") - Expect(k8sClient.Create(ctx, ns)).Should(Succeed()) - - // - By("By creating a new UffizziCluster") - Expect(k8sClient.Create(ctx, uc)).Should(Succeed()) - }) - - It("Should create a HelmRelease and HelmRepository", func() { - // - By("Checking if the HelmRelease was created") - Eventually(func() bool { - if err := k8sClient.Get(ctx, createNamespacesName(helmRelease.Name, ns.Name), helmRelease); err != nil { - return false - } - return true - }) - // - By("Checking if the Loft HelmRepository was created") - Eventually(func() bool { - if err := k8sClient.Get(ctx, createNamespacesName(constants.LOFT_HELM_REPO, ns.Name), helmRepo); err != nil { - return false - } - return true - }) - - }) - - It("Should initialize correctly", func() { - expectedConditions := []metav1.Condition{} - uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) - By("Check if UffizziCluster initializes correctly") - Eventually(func() bool { - if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { - return false - } - expectedConditions = uffizzicluster.GetAllInitializingConditions() - return containsAllConditions(expectedConditions, uc.Status.Conditions) - }, timeout, pollingTimeout).Should(BeTrue()) - d := cmp.Diff(expectedConditions, uc.Status.Conditions) - GinkgoWriter.Printf(diff.PrintWantGot(d)) - }) - - It("Should be in a Ready State", func() { - expectedConditions := []metav1.Condition{} - uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) - By("Check if UffizziCluster has the correct Ready conditions") - Eventually(func() bool { - if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { - return false - } - expectedConditions = uffizzicluster.GetAllReadyConditions() - return containsAllConditions(expectedConditions, uc.Status.Conditions) - }, timeout, pollingTimeout).Should(BeTrue()) - d := cmp.Diff(expectedConditions, uc.Status.Conditions) - GinkgoWriter.Printf(diff.PrintWantGot(d)) - }) - }) - - Context("When putting a cluster to sleep", func() { - It("Should put the cluster to sleep", func() { - By("By putting the UffizziCluster to sleep") - uc.Spec.Sleep = true - Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) - }) - - It("Should be in a Sleep State", func() { - expectedConditions := []metav1.Condition{ - uffizzicluster.APINotReady(), - uffizzicluster.DataStoreNotReady(), - } - uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) - By("Check if UffizziCluster has the correct Sleep conditions") - Eventually(func() bool { - if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { - return false - } - return containsAllConditions(expectedConditions, uc.Status.Conditions) - }, timeout, pollingTimeout).Should(BeTrue()) - d := cmp.Diff(expectedConditions, uc.Status.Conditions) - GinkgoWriter.Printf(diff.PrintWantGot(d)) - }) - }) - - Context("When waking a cluster up", func() { - It("Should wake the cluster up", func() { - By("By waking the UffizziCluster up") - uc.Spec.Sleep = false - Expect(k8sClient.Update(ctx, uc)).Should(Succeed()) - }) +type TestDefinition struct { + Name string + Spec v1alpha1.UffizziClusterSpec +} - It("Should be Awoken", func() { - expectedConditions := []metav1.Condition{ - uffizzicluster.APIReady(), - uffizzicluster.DataStoreReady(), - } - uffizziClusterNSN := createNamespacesName(uc.Name, ns.Name) - By("Check if UffizziCluster has the correct Sleep conditions") - Eventually(func() bool { - if err := k8sClient.Get(ctx, uffizziClusterNSN, uc); err != nil { - return false - } - return containsAllConditions(expectedConditions, uc.Status.Conditions) - }, timeout, pollingTimeout).Should(BeTrue()) - d := cmp.Diff(expectedConditions, uc.Status.Conditions) - GinkgoWriter.Printf(diff.PrintWantGot(d)) - }) - }) +func (td *TestDefinition) ExecLifecycleTest(ctx context.Context) { + ns := resources.CreateTestNamespace(td.Name) + uc := resources.CreateTestUffizziCluster(td.Name, ns.Name) + wrapUffizziClusterLifecycleTest(ctx, ns, uc) +} - Context("When deleting UffizziCluster", func() { - It("Should delete the UffizziCluster", func() { - By("By deleting the UffizziCluster") - Expect(k8sClient.Delete(ctx, uc)).Should(Succeed()) - }) - It("Should delete the Namespace", func() { - By("By deleting the Namespace") - Expect(deleteTestNamespace(ns.Name)).Should(Succeed()) - }) - }) -}) +const ( + timeout = "1m" + pollingTimeout = "100ms" +) -func createNamespacesName(name, namespace string) types.NamespacedName { - return types.NamespacedName{ - Name: name, - Namespace: namespace, +var _ = Describe("Basic UffizziCluster Lifecycle", func() { + ctx := context.Background() + testUffizziCluster := TestDefinition{ + Name: "basic", + Spec: v1alpha1.UffizziClusterSpec{}, } -} - -func deleteTestNamespace(name string) error { - return k8sClient.Delete(ctx, &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - }) -} + // run the testUffizziCluster + testUffizziCluster.ExecLifecycleTest(ctx) +}) diff --git a/src/test/util/common.go b/src/test/util/common.go new file mode 100644 index 00000000..c0da6561 --- /dev/null +++ b/src/test/util/common.go @@ -0,0 +1,19 @@ +package util + +import ( + "math/rand" + "time" +) + +func stringWithCharset(length int, charset string) string { + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, length) + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + return string(b) +} + +func RandomString(length int) string { + return stringWithCharset(length, "abcdefghijklmnopqrstuvwxyz") +} diff --git a/src/test/util/conditions/conditions.go b/src/test/util/conditions/conditions.go new file mode 100644 index 00000000..3b2311ca --- /dev/null +++ b/src/test/util/conditions/conditions.go @@ -0,0 +1,30 @@ +package conditions + +import ( + "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util/diff" + "github.com/google/go-cmp/cmp" + "github.com/onsi/ginkgo/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Checks if the required conditions are present and match in the actual conditions slice. +// Both requiredConditions and actualConditions are slices of metav1.Condition. +func ContainsAllConditions(requiredConditions, actualConditions []metav1.Condition) bool { + for _, requiredCondition := range requiredConditions { + found := false + for _, actualCondition := range actualConditions { + if actualCondition.Type == requiredCondition.Type && + actualCondition.Status == requiredCondition.Status { + // Add more condition checks here if necessary (e.g., Reason, Message) + found = true + d := cmp.Diff(requiredConditions, actualConditions) + ginkgo.GinkgoWriter.Printf(diff.PrintWantGot(d)) + break + } + } + if !found { + return false + } + } + return true +} diff --git a/src/test/util/resources/resources.go b/src/test/util/resources/resources.go new file mode 100644 index 00000000..827c005c --- /dev/null +++ b/src/test/util/resources/resources.go @@ -0,0 +1,61 @@ +package resources + +import ( + "github.com/UffizziCloud/uffizzi-cluster-operator/src/api/v1alpha1" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/pkg/constants" + "github.com/UffizziCloud/uffizzi-cluster-operator/src/test/util" + "github.com/fluxcd/helm-controller/api/v2beta1" + "github.com/fluxcd/source-controller/api/v1beta2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "strings" +) + +func CreateTestNamespace(name string) *v1.Namespace { + return &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: strings.Join([]string{name, util.RandomString(5)}, "-"), + }, + } +} + +func CreateTestUffizziCluster(name, ns string) *v1alpha1.UffizziCluster { + return &v1alpha1.UffizziCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } +} + +func CreateTestUffizziClusterWithSpec(name, ns string, spec v1alpha1.UffizziClusterSpec) *v1alpha1.UffizziCluster { + uc := CreateTestUffizziCluster(name, ns) + uc.Spec = spec + return uc +} + +func GetHelmReleaseFromUffizziCluster(uc *v1alpha1.UffizziCluster) *v2beta1.HelmRelease { + return &v2beta1.HelmRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "uc-" + uc.Name, + Namespace: uc.Namespace, + }, + } +} + +func GetHelmRepositoryFromUffizziCluster(uc *v1alpha1.UffizziCluster) *v1beta2.HelmRepository { + return &v1beta2.HelmRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.LOFT_HELM_REPO, + Namespace: uc.Namespace, + }, + } +} + +func CreateNamespacedName(name, namespace string) types.NamespacedName { + return types.NamespacedName{ + Name: name, + Namespace: namespace, + } +}