diff --git a/docs/TektonPipeline.md b/docs/TektonPipeline.md index 979b11e891..9667df83e4 100644 --- a/docs/TektonPipeline.md +++ b/docs/TektonPipeline.md @@ -56,6 +56,7 @@ spec: threads-per-controller: 2 kube-api-qps: 5.0 kube-api-burst: 10 + statefulset-ordinals: false options: disabled: false configMaps: {} @@ -263,6 +264,7 @@ spec: threads-per-controller: 2 kube-api-qps: 5.0 kube-api-burst: 10 + statefulset-ordinals: false ``` These fields are optional and there is no default values. If user passes them, operator will include most of fields into the deployment `tekton-pipelines-controller` under the container `tekton-pipelines-controller` as arguments(duplicate name? No, container and deployment has the same name), otherwise pipelines controller's default values will be considered. and `buckets` field is updated into `config-leader-election` config-map under the namespace `tekton-pipelines`. @@ -275,6 +277,11 @@ A high level descriptions are given here. To get the detailed information please * `threads-per-controller` - is the number of threads(aka worker) to use when processing the pipelines controller's workqueue, default value in pipelines controller is `2` * `kube-api-qps` - QPS indicates the maximum QPS to the cluster master from the REST client, default value in pipeline controller is `5.0` * `kube-api-burst` - maximum burst for throttle, default value in pipeline controller is `10` +* `statefulset-ordinals` - enables StatefulSet Ordinals mode for the Tekton Pipelines Controller. When set to true, the Pipelines Controller is deployed as a StatefulSet, allowing for multiple replicas to be configured with a load-balancing mode. This ensures that the load is evenly distributed across replicas, and the number of buckets is enforced to match the number of replicas. +Moreover, There are two mechanisms available for scaling for scaling Pipelines Controller horizontally: +- Using leader election, which allows for failover, but can result in hot-spotting. +- Using StatefulSet ordinals, which doesn't allow for failover, but guarantees load is evenly spread across replicas. + > #### Note: > * `kube-api-qps` and `kube-api-burst` will be multiplied by 2 in pipelines controller. To get the detailed information visit [Performance Configuration](https://tekton.dev/docs/pipelines/tekton-controller-performance-configuration/) guide diff --git a/pkg/apis/operator/v1alpha1/tektonpipeline_defaults.go b/pkg/apis/operator/v1alpha1/tektonpipeline_defaults.go index 142e6f4158..83563a7347 100644 --- a/pkg/apis/operator/v1alpha1/tektonpipeline_defaults.go +++ b/pkg/apis/operator/v1alpha1/tektonpipeline_defaults.go @@ -150,6 +150,15 @@ func (p *Pipeline) setDefaults() { p.EnableGitResolver = ptr.Bool(true) } + // Statefulset Ordinals + // if StatefulSet Ordinals mode, buckets should be equal to replicas + if p.Performance.StatefulsetOrdinals != nil && *p.Performance.StatefulsetOrdinals { + if p.Performance.Replicas != nil && *p.Performance.Replicas > 1 { + replicas := uint(*p.Performance.Replicas) + p.Performance.Buckets = &replicas + } + } + // run platform specific defaulting if IsOpenShiftPlatform() { p.openshiftDefaulting() diff --git a/pkg/apis/operator/v1alpha1/tektonpipeline_types.go b/pkg/apis/operator/v1alpha1/tektonpipeline_types.go index 4a9c1156be..ea0872fa0e 100644 --- a/pkg/apis/operator/v1alpha1/tektonpipeline_types.go +++ b/pkg/apis/operator/v1alpha1/tektonpipeline_types.go @@ -189,6 +189,8 @@ type PipelinePerformanceProperties struct { // +optional PipelinePerformanceLeaderElectionConfig `json:",inline"` // +optional + PipelinePerformanceStatefulsetOrdinalsConfig `json:",inline"` + // +optional PipelineDeploymentPerformanceArgs `json:",inline"` // +optional Replicas *int32 `json:"replicas,omitempty"` @@ -201,6 +203,12 @@ type PipelinePerformanceLeaderElectionConfig struct { Buckets *uint `json:"buckets,omitempty"` } +// allow to configure pipelines controller ha mode to statefulset ordinals +type PipelinePerformanceStatefulsetOrdinalsConfig struct { + //if is true, enable StatefulsetOrdinals mode + StatefulsetOrdinals *bool `json:"statefulset-ordinals,omitempty"` +} + // performance configurations to tune the performance of the pipeline controller // these properties will be added/updated as arguments in pipeline controller deployment // https://tekton.dev/docs/pipelines/tekton-controller-performance-configuration/ diff --git a/pkg/apis/operator/v1alpha1/tektonpipeline_validation.go b/pkg/apis/operator/v1alpha1/tektonpipeline_validation.go index 9df3b45e94..d9185554d4 100644 --- a/pkg/apis/operator/v1alpha1/tektonpipeline_validation.go +++ b/pkg/apis/operator/v1alpha1/tektonpipeline_validation.go @@ -116,5 +116,15 @@ func (prof *PipelinePerformanceProperties) validate(path string) *apis.FieldErro } } + // check for StatefulsetOrdinals and Replicas + if prof.StatefulsetOrdinals != nil && *prof.StatefulsetOrdinals { + if prof.Replicas != nil { + replicas := uint(*prof.Replicas) + if *prof.Buckets != replicas { + errs = errs.Also(apis.ErrInvalidValue(*prof.Replicas, fmt.Sprintf("%s.replicas", path), "spec.performance.replicas must equal spec.performance.buckets for statefulset ordinals")) + } + } + } + return errs } diff --git a/pkg/apis/operator/v1alpha1/tektonpipeline_validation_test.go b/pkg/apis/operator/v1alpha1/tektonpipeline_validation_test.go index 1a2fda89e7..d90304e2b1 100644 --- a/pkg/apis/operator/v1alpha1/tektonpipeline_validation_test.go +++ b/pkg/apis/operator/v1alpha1/tektonpipeline_validation_test.go @@ -283,6 +283,13 @@ func TestTektonPipelinePerformancePropertiesValidate(t *testing.T) { return &value } + // return pointer value for replicas + getReplicas := func(value int32) *int32 { + return &value + } + + statefulsetOrdinals := true + // validate buckets minimum range tp.Spec.PipelineProperties.Performance = PipelinePerformanceProperties{} tp.Spec.PipelineProperties.Performance.DisableHA = false @@ -310,4 +317,26 @@ func TestTektonPipelinePerformancePropertiesValidate(t *testing.T) { tp.Spec.PipelineProperties.Performance.Buckets = getBuckets(10) errs = tp.Validate(context.TODO()) assert.Equal(t, "", errs.Error()) + + // validate buckets is equal to replicas when StatefulsetOrdinals is true + tp.Spec.PipelineProperties.Performance = PipelinePerformanceProperties{} + tp.Spec.PipelineProperties.Performance.DisableHA = false + bucketValue := uint(5) + tp.Spec.PipelineProperties.Performance.Buckets = getBuckets(bucketValue) + replicaValue := int32(5) + tp.Spec.PipelineProperties.Performance.Replicas = getReplicas(replicaValue) + tp.Spec.PipelineProperties.Performance.StatefulsetOrdinals = &statefulsetOrdinals + errs = tp.Validate(context.TODO()) + assert.Equal(t, "", errs.Error()) + + // validate error when buckets is not equal to replica + tp.Spec.PipelineProperties.Performance = PipelinePerformanceProperties{} + tp.Spec.PipelineProperties.Performance.DisableHA = false + tp.Spec.PipelineProperties.Performance.StatefulsetOrdinals = &statefulsetOrdinals + bucketValue = uint(5) + tp.Spec.PipelineProperties.Performance.Buckets = getBuckets(bucketValue) + tp.Spec.PipelineProperties.Performance.Replicas = getReplicas(3) + errs = tp.Validate(context.TODO()) + expectedErrorMessage := "invalid value: 3: spec.performance.replicas\nspec.performance.replicas must equal spec.performance.buckets for statefulset ordinals" + assert.Equal(t, expectedErrorMessage, errs.Error()) } diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index c28286fe04..06630e684f 100644 --- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -896,6 +896,7 @@ func (in *PipelinePerformanceLeaderElectionConfig) DeepCopy() *PipelinePerforman func (in *PipelinePerformanceProperties) DeepCopyInto(out *PipelinePerformanceProperties) { *out = *in in.PipelinePerformanceLeaderElectionConfig.DeepCopyInto(&out.PipelinePerformanceLeaderElectionConfig) + in.PipelinePerformanceStatefulsetOrdinalsConfig.DeepCopyInto(&out.PipelinePerformanceStatefulsetOrdinalsConfig) in.PipelineDeploymentPerformanceArgs.DeepCopyInto(&out.PipelineDeploymentPerformanceArgs) if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas @@ -915,6 +916,27 @@ func (in *PipelinePerformanceProperties) DeepCopy() *PipelinePerformanceProperti return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PipelinePerformanceStatefulsetOrdinalsConfig) DeepCopyInto(out *PipelinePerformanceStatefulsetOrdinalsConfig) { + *out = *in + if in.StatefulsetOrdinals != nil { + in, out := &in.StatefulsetOrdinals, &out.StatefulsetOrdinals + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PipelinePerformanceStatefulsetOrdinalsConfig. +func (in *PipelinePerformanceStatefulsetOrdinalsConfig) DeepCopy() *PipelinePerformanceStatefulsetOrdinalsConfig { + if in == nil { + return nil + } + out := new(PipelinePerformanceStatefulsetOrdinalsConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PipelineProperties) DeepCopyInto(out *PipelineProperties) { *out = *in diff --git a/pkg/reconciler/common/transformers.go b/pkg/reconciler/common/transformers.go index 4ad661450d..c2bd37e3f0 100644 --- a/pkg/reconciler/common/transformers.go +++ b/pkg/reconciler/common/transformers.go @@ -31,6 +31,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "knative.dev/pkg/logging" @@ -1013,3 +1014,87 @@ func AddSecretData(data map[string][]byte, annotations map[string]string) mf.Tra return nil } } + +// ConvertDeploymentToStatefulSet converts a Deployment to a StatefulSet with given parameters +func ConvertDeploymentToStatefulSet(controllerName, serviceName string) mf.Transformer { + return func(u *unstructured.Unstructured) error { + if u.GetKind() != "Deployment" || u.GetName() != controllerName { + return nil + } + + d := &appsv1.Deployment{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, d) + if err != nil { + return err + } + + ss := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: appsv1.SchemeGroupVersion.Group + "/" + appsv1.SchemeGroupVersion.Version, + }, + ObjectMeta: d.ObjectMeta, + Spec: appsv1.StatefulSetSpec{ + Selector: d.Spec.Selector, + ServiceName: serviceName, + Template: d.Spec.Template, + Replicas: d.Spec.Replicas, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + }, + } + + unstrObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ss) + if err != nil { + return err + } + + u.SetUnstructuredContent(unstrObj) + + return nil + } +} + +// AddStatefulEnvVars adds environment variables to the statefulset based on given parameters +func AddStatefulEnvVars(controllerName, serviceName, statefulServiceEnvVar, controllerOrdinalEnvVar string) mf.Transformer { + return func(u *unstructured.Unstructured) error { + if u.GetKind() != "StatefulSet" || u.GetName() != controllerName { + return nil + } + + ss := &appsv1.StatefulSet{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, ss) + if err != nil { + return err + } + + newEnvVars := []corev1.EnvVar{ + { + Name: statefulServiceEnvVar, + Value: serviceName, + }, + { + Name: controllerOrdinalEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + } + + if len(ss.Spec.Template.Spec.Containers) > 0 { + ss.Spec.Template.Spec.Containers[0].Env = append(ss.Spec.Template.Spec.Containers[0].Env, newEnvVars...) + } + + unstrObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ss) + if err != nil { + return err + } + + u.SetUnstructuredContent(unstrObj) + + return nil + } +} diff --git a/pkg/reconciler/kubernetes/tektoninstallerset/client/check.go b/pkg/reconciler/kubernetes/tektoninstallerset/client/check.go index 9521bb6756..39e7ed4616 100644 --- a/pkg/reconciler/kubernetes/tektoninstallerset/client/check.go +++ b/pkg/reconciler/kubernetes/tektoninstallerset/client/check.go @@ -98,7 +98,9 @@ func verifyMainInstallerSets(iSets []v1alpha1.TektonInstallerSet) error { static = true } if strings.Contains(iSets[0].GetName(), InstallerSubTypeDeployment) || - strings.Contains(iSets[1].GetName(), InstallerSubTypeDeployment) { + strings.Contains(iSets[1].GetName(), InstallerSubTypeDeployment) || + strings.Contains(iSets[0].GetName(), InstallerSubTypeStatefulset) || + strings.Contains(iSets[1].GetName(), InstallerSubTypeStatefulset) { deployment = true } if !(static && deployment) { diff --git a/pkg/reconciler/kubernetes/tektoninstallerset/client/cleanup.go b/pkg/reconciler/kubernetes/tektoninstallerset/client/cleanup.go index efebe84325..88e9094c4e 100644 --- a/pkg/reconciler/kubernetes/tektoninstallerset/client/cleanup.go +++ b/pkg/reconciler/kubernetes/tektoninstallerset/client/cleanup.go @@ -57,7 +57,8 @@ func (i *InstallerSetClient) CleanupMainSet(ctx context.Context) error { // now delete all deployment installerSet for _, is := range list.Items { - if strings.Contains(is.GetName(), InstallerSubTypeDeployment) { + if strings.Contains(is.GetName(), InstallerSubTypeDeployment) || + strings.Contains(is.GetName(), InstallerSubTypeStatefulset) { logger.Debugf("deleting main-deployment installer set: %s", is.GetName()) err = i.clientSet.Delete(ctx, is.GetName(), metav1.DeleteOptions{ PropagationPolicy: &deletePropagationPolicy, @@ -125,3 +126,37 @@ func (i *InstallerSetClient) cleanup(ctx context.Context, isType string) error { } return nil } + +func (i *InstallerSetClient) CleanupSubTypeDeployment(ctx context.Context) error { + return i.cleanupSubType(ctx, InstallerTypeMain, InstallerSubTypeDeployment) +} + +func (i *InstallerSetClient) CleanupSubTypeStatefulset(ctx context.Context) error { + return i.cleanupSubType(ctx, InstallerTypeMain, InstallerSubTypeStatefulset) +} + +func (i *InstallerSetClient) cleanupSubType(ctx context.Context, isType string, isSubType string) error { + logger := logging.FromContext(ctx).With("kind", i.resourceKind, "type", isType) + + list, err := i.clientSet.List(ctx, metav1.ListOptions{LabelSelector: i.getSetLabels(isType)}) + if err != nil { + return err + } + + if len(list.Items) != 1 { + logger.Errorf("found more than 1 installerSet for %s something fishy, cleaning up all", isType) + } + + for _, is := range list.Items { + if strings.Contains(is.GetName(), isSubType) { + logger.Debugf("deleting %s installer set: %s", isType, is.GetName()) + err = i.clientSet.Delete(ctx, is.GetName(), metav1.DeleteOptions{ + PropagationPolicy: &deletePropagationPolicy, + }) + if err != nil { + return fmt.Errorf("failed to delete %s set: %s", isType, is.GetName()) + } + } + } + return nil +} diff --git a/pkg/reconciler/kubernetes/tektoninstallerset/client/client.go b/pkg/reconciler/kubernetes/tektoninstallerset/client/client.go index 0da022430b..f4939afd03 100644 --- a/pkg/reconciler/kubernetes/tektoninstallerset/client/client.go +++ b/pkg/reconciler/kubernetes/tektoninstallerset/client/client.go @@ -26,8 +26,9 @@ import ( ) const ( - InstallerSubTypeStatic = "static" - InstallerSubTypeDeployment = "deployment" + InstallerSubTypeStatic = "static" + InstallerSubTypeDeployment = "deployment" + InstallerSubTypeStatefulset = "statefulset" InstallerTypeMain = "main" InstallerTypePre = "pre" diff --git a/pkg/reconciler/kubernetes/tektoninstallerset/client/create.go b/pkg/reconciler/kubernetes/tektoninstallerset/client/create.go index 48a7ae4ac5..556420b5e2 100644 --- a/pkg/reconciler/kubernetes/tektoninstallerset/client/create.go +++ b/pkg/reconciler/kubernetes/tektoninstallerset/client/create.go @@ -60,6 +60,7 @@ func (i *InstallerSetClient) create(ctx context.Context, comp v1alpha1.TektonCom func (i *InstallerSetClient) makeMainSets(ctx context.Context, comp v1alpha1.TektonComponent, manifest *mf.Manifest) ([]v1alpha1.TektonInstallerSet, error) { staticManifest := manifest.Filter(mf.Not(mf.ByKind("Deployment")), mf.Not(mf.ByKind("Service"))) deploymentManifest := manifest.Filter(mf.Any(mf.ByKind("Deployment"), mf.ByKind("Service"))) + statefulSetManifest := manifest.Filter(mf.Any(mf.ByKind("StatefulSet"), mf.ByKind("Service"))) kind := strings.ToLower(strings.TrimPrefix(i.resourceKind, "Tekton")) staticName := fmt.Sprintf("%s-%s-%s-", kind, InstallerTypeMain, InstallerSubTypeStatic) @@ -78,6 +79,7 @@ func (i *InstallerSetClient) makeMainSets(ctx context.Context, comp v1alpha1.Tek } deployName := fmt.Sprintf("%s-%s-%s-", kind, InstallerTypeMain, InstallerSubTypeDeployment) + deploymentIS, err := i.makeInstallerSet(ctx, comp, &deploymentManifest, deployName, InstallerTypeMain, nil) if err != nil { return nil, err @@ -87,6 +89,25 @@ func (i *InstallerSetClient) makeMainSets(ctx context.Context, comp v1alpha1.Tek if err != nil { return nil, err } + + statefulSet := false + if pipeline, ok := comp.(*v1alpha1.TektonPipeline); ok { + statefulSet = pipeline.Spec.Performance.StatefulsetOrdinals != nil && *pipeline.Spec.Performance.StatefulsetOrdinals + } + if statefulSet { + stsName := fmt.Sprintf("%s-%s-%s-", kind, InstallerTypeMain, InstallerSubTypeStatefulset) + stsIS, err := i.makeInstallerSet(ctx, comp, &statefulSetManifest, stsName, InstallerTypeMain, nil) + if err != nil { + return nil, err + } + + stsIS, err = i.clientSet.Create(ctx, stsIS, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return []v1alpha1.TektonInstallerSet{*staticIS, *deploymentIS, *stsIS}, nil + } + return []v1alpha1.TektonInstallerSet{*staticIS, *deploymentIS}, nil } diff --git a/pkg/reconciler/kubernetes/tektonpipeline/reconcile.go b/pkg/reconciler/kubernetes/tektonpipeline/reconcile.go index 51df2e9a2f..6815027f1e 100644 --- a/pkg/reconciler/kubernetes/tektonpipeline/reconcile.go +++ b/pkg/reconciler/kubernetes/tektonpipeline/reconcile.go @@ -84,6 +84,20 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, tp *v1alpha1.TektonPipel return err } + // Pipeline controller is deployed as statefulset, ensure deployment installerset is deleted + if tp.Spec.Performance.StatefulsetOrdinals != nil && *tp.Spec.Performance.StatefulsetOrdinals { + if err := r.installerSetClient.CleanupSubTypeDeployment(ctx); err != nil { + logger.Error("failed to delete main deployment installer set: %v", err) + return err + } + } else { + // Pipeline controller is deployed as deployment, ensure statefulset installerset is deleted + if err := r.installerSetClient.CleanupSubTypeStatefulset(ctx); err != nil { + logger.Error("failed to delete main statefulset installer set: %v", err) + return err + } + } + if err := r.extension.PreReconcile(ctx, tp); err != nil { msg := fmt.Sprintf("PreReconciliation failed: %s", err.Error()) logger.Error(msg) diff --git a/pkg/reconciler/kubernetes/tektonpipeline/transform.go b/pkg/reconciler/kubernetes/tektonpipeline/transform.go index 634f44761a..ac6de37716 100644 --- a/pkg/reconciler/kubernetes/tektonpipeline/transform.go +++ b/pkg/reconciler/kubernetes/tektonpipeline/transform.go @@ -50,6 +50,11 @@ const ( pipelinesRemoteResolverControllerContainer = "controller" resolverEnvKeyTektonHubApi = "tekton-hub-api" resolverEnvKeyArtifactHubApi = "artifact-hub-api" + + tektonPipelinesControllerName = "tekton-pipelines-controller" + tektonPipelinesServiceName = "tekton-pipelines-controller" + tektonPipelinesControllerStatefulServiceName = "STATEFUL_SERVICE_NAME" + tektonPipelinesControllerStatefulControllerOrdinal = "STATEFUL_CONTROLLER_ORDINAL" ) func filterAndTransform(extension common.Extension) client.FilterAndTransform { @@ -82,6 +87,11 @@ func filterAndTransform(extension common.Extension) client.FilterAndTransform { updatePerformanceFlagsInDeployment(pipeline), updateResolverConfigEnvironmentsInDeployment(pipeline), } + if pipeline.Spec.Performance.StatefulsetOrdinals != nil && *pipeline.Spec.Performance.StatefulsetOrdinals { + extra = append(extra, common.ConvertDeploymentToStatefulSet(tektonPipelinesControllerName, tektonPipelinesServiceName), common.AddStatefulEnvVars( + tektonPipelinesControllerName, tektonPipelinesServiceName, tektonPipelinesControllerStatefulServiceName, tektonPipelinesControllerStatefulControllerOrdinal)) + } + trns = append(trns, extra...) if err := common.Transform(ctx, manifest, instance, trns...); err != nil { diff --git a/test/e2e/common/07_tektonpipelinestatefulset_test.go b/test/e2e/common/07_tektonpipelinestatefulset_test.go new file mode 100644 index 0000000000..6452a88858 --- /dev/null +++ b/test/e2e/common/07_tektonpipelinestatefulset_test.go @@ -0,0 +1,76 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "os" + "testing" + + "github.com/tektoncd/operator/test/utils" + + "github.com/tektoncd/operator/test/client" + + "github.com/tektoncd/operator/test/resources" +) + +// TestTektonPipelinesStatefulset verifies the TektonPipelines creation, statefulset recreation, and TektonPipelines deletion. +func TestTektonPipelinesStatefulset(t *testing.T) { + crNames := utils.ResourceNames{ + TektonConfig: "config", + TektonPipeline: "pipeline", + TargetNamespace: "tekton-pipelines", + } + + if os.Getenv("TARGET") == "openshift" { + crNames.TargetNamespace = "openshift-pipelines" + } + + clients := client.Setup(t, crNames.TargetNamespace) + + utils.CleanupOnInterrupt(func() { utils.TearDownPipeline(clients, crNames.TektonPipeline) }) + utils.CleanupOnInterrupt(func() { utils.TearDownNamespace(clients, crNames.TargetNamespace) }) + defer utils.TearDownNamespace(clients, crNames.TargetNamespace) + defer utils.TearDownPipeline(clients, crNames.TektonPipeline) + + resources.EnsureNoTektonConfigInstance(t, clients, crNames) + + // Create a TektonPipeline + if _, err := resources.EnsureTektonPipelineWithStatefulsetExists(clients.TektonPipeline(), crNames); err != nil { + t.Fatalf("TektonPipeline %q failed to create: %v", crNames.TektonPipeline, err) + } + + // Test if TektonPipeline can reach the READY status + t.Run("create-pipeline", func(t *testing.T) { + resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames) + }) + + // Delete the statefulsets one by one to see if they will be recreated. + t.Run("restore-pipeline-statefulset", func(t *testing.T) { + resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames) + resources.DeleteAndVerifyStatefulSet(t, clients, crNames.TargetNamespace, utils.TektonPipelineDeploymentLabel) + resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames) + }) + + // Delete the TektonPipeline CR instance to see if all resources will be removed + t.Run("delete-pipeline", func(t *testing.T) { + resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames) + resources.TektonPipelineCRDelete(t, clients, crNames) + }) +} diff --git a/test/resources/statefulset.go b/test/resources/statefulset.go new file mode 100644 index 0000000000..11767edf7e --- /dev/null +++ b/test/resources/statefulset.go @@ -0,0 +1,78 @@ +/* +Copyright 2024 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/tektoncd/operator/test/utils" +) + +// DeleteAndVerifyStatefulSet verifies whether the Tekton Pipelines StatefulSet controller +// is able to recreate its pods after being deleted. +func DeleteAndVerifyStatefulSet(t *testing.T, clients *utils.Clients, namespace, labelSelector string) { + listOptions := metav1.ListOptions{LabelSelector: labelSelector} + stsList, err := clients.KubeClient.AppsV1().StatefulSets(namespace).List(context.TODO(), listOptions) + if err != nil { + t.Fatalf("Failed to get any StatefulSet under the namespace %q: %v", namespace, err) + } + if len(stsList.Items) == 0 { + t.Fatalf("No StatefulSet under the namespace %q was found", namespace) + } + + // Delete the first StatefulSet and verify the operator recreates it + statefulSet := stsList.Items[0] + err = clients.KubeClient.AppsV1().StatefulSets(statefulSet.Namespace).Delete(context.TODO(), statefulSet.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete StatefulSet %s/%s: %v", statefulSet.Namespace, statefulSet.Name, err) + } + + // Poll and wait for the StatefulSet to be recreated and ready + waitErr := wait.PollUntilContextTimeout(context.TODO(), utils.Interval, utils.Timeout, true, func(ctx context.Context) (bool, error) { + sts, err := clients.KubeClient. + AppsV1().StatefulSets(statefulSet.Namespace).Get(context.TODO(), statefulSet.Name, metav1.GetOptions{}) + if err != nil { + // If the StatefulSet is not found, we continue to wait for it to be recreated. + if apierrs.IsNotFound(err) { + return false, nil + } + return false, err + } + + // Check if the StatefulSet is available + return IsStatefulSetAvailable(sts) + }) + + if waitErr != nil { + t.Fatalf("The StatefulSet %s/%s failed to reach the desired state: %v", statefulSet.Namespace, statefulSet.Name, waitErr) + } +} + +// IsStatefulSetAvailable checks if a StatefulSet is available by verifying its ReadyReplicas +func IsStatefulSetAvailable(sts *appsv1.StatefulSet) (bool, error) { + // Check if the number of ready replicas matches the desired replicas + if sts.Status.ReadyReplicas == *sts.Spec.Replicas { + return true, nil + } + return false, nil +} diff --git a/test/resources/tektonpipelines.go b/test/resources/tektonpipelines.go index 47d899c136..48f08831ef 100644 --- a/test/resources/tektonpipelines.go +++ b/test/resources/tektonpipelines.go @@ -142,3 +142,37 @@ func verifyNoTektonPipelineCR(clients *utils.Clients) error { } return nil } + +// EnsureTektonPipelineWithStatefulsetExists creates a TektonPipeline with the name names.TektonPipeline, if it does not exist. +func EnsureTektonPipelineWithStatefulsetExists(clients pipelinev1alpha1.TektonPipelineInterface, names utils.ResourceNames) (*v1alpha1.TektonPipeline, error) { + // If this function is called by the upgrade tests, we only create the custom resource if it does not exist. + tpCR, err := clients.Get(context.TODO(), names.TektonPipeline, metav1.GetOptions{}) + if err == nil { + return tpCR, err + } + if apierrs.IsNotFound(err) { + statefulsetOrdinals := true + + tpCR = &v1alpha1.TektonPipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.TektonPipeline, + }, + Spec: v1alpha1.TektonPipelineSpec{ + CommonSpec: v1alpha1.CommonSpec{ + TargetNamespace: names.TargetNamespace, + }, + Pipeline: v1alpha1.Pipeline{ + PipelineProperties: v1alpha1.PipelineProperties{ + Performance: v1alpha1.PipelinePerformanceProperties{ + PipelinePerformanceStatefulsetOrdinalsConfig: v1alpha1.PipelinePerformanceStatefulsetOrdinalsConfig{ + StatefulsetOrdinals: &statefulsetOrdinals, + }, + }, + }, + }, + }, + } + return clients.Create(context.TODO(), tpCR, metav1.CreateOptions{}) + } + return tpCR, err +}