From 573930a432a2a82641c11f07c7308786e2d5c3b8 Mon Sep 17 00:00:00 2001 From: AiRanthem Date: Thu, 2 Jan 2025 19:27:18 +0800 Subject: [PATCH 1/4] feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem --- apis/apps/v1beta1/statefulset_types.go | 39 +++++++- apis/apps/v1beta1/zz_generated.deepcopy.go | 21 ++++- .../bases/apps.kruise.io_statefulsets.yaml | 6 +- .../apps.kruise.io_uniteddeployments.yaml | 6 +- .../persistent_pod_state_controller.go | 9 +- .../persistent_pod_state_controller_test.go | 17 ++-- .../statefulset/stateful_pod_control_test.go | 21 ++++- .../statefulset/stateful_set_control_test.go | 9 +- .../statefulset/stateful_set_utils.go | 6 +- .../statefulset/stateful_set_utils_test.go | 91 ++++++++++++------- .../controllerfinder/controller_finder.go | 4 +- .../validating/statefulset_validation.go | 20 ++-- .../validating/statefulset_validation_test.go | 8 +- test/e2e/apps/statefulset.go | 34 ++++++- 14 files changed, 217 insertions(+), 74 deletions(-) diff --git a/apis/apps/v1beta1/statefulset_types.go b/apis/apps/v1beta1/statefulset_types.go index 32d408b00d..5a429df981 100644 --- a/apis/apps/v1beta1/statefulset_types.go +++ b/apis/apps/v1beta1/statefulset_types.go @@ -17,10 +17,15 @@ limitations under the License. package v1beta1 import ( + "strconv" + "strings" + apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" appspub "github.com/openkruise/kruise/apis/apps/pub" ) @@ -275,7 +280,8 @@ type StatefulSetSpec struct { // Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3]) // - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2. // Then controller will delete Pod-1 (existing Pods will be [0, 2]) - ReserveOrdinals []int `json:"reserveOrdinals,omitempty"` + // You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5]. + ReserveOrdinals ReserveOrdinal `json:"reserveOrdinals,omitempty"` // Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update. Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"` @@ -299,6 +305,37 @@ type StatefulSetSpec struct { Ordinals *StatefulSetOrdinals `json:"ordinals,omitempty"` } +type ReserveOrdinal []intstr.IntOrString + +func (r ReserveOrdinal) GetIntSet() sets.Set[int] { + values := sets.New[int]() + for _, elem := range r { + if elem.Type == intstr.Int { + values.Insert(int(elem.IntVal)) + } else { + split := strings.Split(elem.StrVal, "-") + if len(split) != 2 { + klog.ErrorS(nil, "invalid range reserveOrdinal found, an empty slice will be returned", "reserveOrdinal", elem.StrVal) + return nil + } + start, err := strconv.Atoi(split[0]) + if err != nil { + klog.ErrorS(err, "invalid range reserveOrdinal found, an empty slice will be returned", "reserveOrdinal", elem.StrVal) + return nil + } + end, err := strconv.Atoi(split[1]) + if err != nil { + klog.ErrorS(err, "invalid range reserveOrdinal found, an empty slice will be returned", "reserveOrdinal", elem.StrVal) + return nil + } + for i := start; i <= end; i++ { + values.Insert(i) + } + } + } + return values +} + // StatefulSetScaleStrategy defines strategies for pods scale. type StatefulSetScaleStrategy struct { // The maximum number of pods that can be unavailable during scaling. diff --git a/apis/apps/v1beta1/zz_generated.deepcopy.go b/apis/apps/v1beta1/zz_generated.deepcopy.go index 90d33e8432..f2cd414666 100644 --- a/apis/apps/v1beta1/zz_generated.deepcopy.go +++ b/apis/apps/v1beta1/zz_generated.deepcopy.go @@ -29,6 +29,25 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ReserveOrdinal) DeepCopyInto(out *ReserveOrdinal) { + { + in := &in + *out = make(ReserveOrdinal, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReserveOrdinal. +func (in ReserveOrdinal) DeepCopy() ReserveOrdinal { + if in == nil { + return nil + } + out := new(ReserveOrdinal) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RollingUpdateStatefulSetStrategy) DeepCopyInto(out *RollingUpdateStatefulSetStrategy) { *out = *in @@ -208,7 +227,7 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { } if in.ReserveOrdinals != nil { in, out := &in.ReserveOrdinals, &out.ReserveOrdinals - *out = make([]int, len(*in)) + *out = make(ReserveOrdinal, len(*in)) copy(*out, *in) } if in.Lifecycle != nil { diff --git a/config/crd/bases/apps.kruise.io_statefulsets.yaml b/config/crd/bases/apps.kruise.io_statefulsets.yaml index 8794b66135..e6dbedc46c 100644 --- a/config/crd/bases/apps.kruise.io_statefulsets.yaml +++ b/config/crd/bases/apps.kruise.io_statefulsets.yaml @@ -652,8 +652,12 @@ spec: Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3]) - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2. Then controller will delete Pod-1 (existing Pods will be [0, 2]) + You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5]. items: - type: integer + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: array revisionHistoryLimit: description: |- diff --git a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml index e03c3c0474..104fe278b9 100644 --- a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml +++ b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml @@ -275,8 +275,12 @@ spec: Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3]) - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2. Then controller will delete Pod-1 (existing Pods will be [0, 2]) + You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5]. items: - type: integer + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: array revisionHistoryLimit: description: |- diff --git a/pkg/controller/persistentpodstate/persistent_pod_state_controller.go b/pkg/controller/persistentpodstate/persistent_pod_state_controller.go index 46d1c6659f..a5dbed50dc 100644 --- a/pkg/controller/persistentpodstate/persistent_pod_state_controller.go +++ b/pkg/controller/persistentpodstate/persistent_pod_state_controller.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/openkruise/kruise/pkg/util/configuration" + "k8s.io/utils/ptr" ctrlUtil "github.com/openkruise/kruise/pkg/controller/util" @@ -36,7 +37,6 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/klog/v2" podutil "k8s.io/kubernetes/pkg/api/v1/pod" - utilpointer "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -146,7 +146,7 @@ type innerStatefulset struct { // replicas Replicas int32 // kruise statefulset filed - ReserveOrdinals []int + ReserveOrdinals sets.Set[int] DeletionTimestamp *metav1.Time } @@ -354,11 +354,10 @@ func (r *ReconcilePersistentPodState) getPodState(pod *corev1.Pod, nodeTopologyK } func isInStatefulSetReplicas(index int, sts *innerStatefulset) bool { - reserveOrdinals := sets.NewInt(sts.ReserveOrdinals...) replicas := sets.NewInt() replicaIndex := 0 for realReplicaCount := 0; realReplicaCount < int(sts.Replicas); replicaIndex++ { - if reserveOrdinals.Has(replicaIndex) { + if sts.ReserveOrdinals.Has(replicaIndex) { continue } realReplicaCount++ @@ -457,7 +456,7 @@ func newStatefulSetPersistentPodState(workload *controllerfinder.ScaleAndSelecto APIVersion: workload.APIVersion, Kind: workload.Kind, Name: workload.Name, - Controller: utilpointer.BoolPtr(true), + Controller: ptr.To(true), UID: workload.UID, }, }, diff --git a/pkg/controller/persistentpodstate/persistent_pod_state_controller_test.go b/pkg/controller/persistentpodstate/persistent_pod_state_controller_test.go index 6f6ed4e9ca..83a0b790e9 100644 --- a/pkg/controller/persistentpodstate/persistent_pod_state_controller_test.go +++ b/pkg/controller/persistentpodstate/persistent_pod_state_controller_test.go @@ -23,7 +23,9 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/utils/ptr" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1" @@ -36,7 +38,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/apis/apps" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -56,7 +57,7 @@ var ( UID: "012d18d5-5eb9-449d-b670-3da8fec8852f", }, Spec: appsv1beta1.StatefulSetSpec{ - Replicas: pointer.Int32Ptr(10), + Replicas: ptr.To[int32](10), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, @@ -70,7 +71,7 @@ var ( Namespace: "ns-test", OwnerReferences: []metav1.OwnerReference{ { - Controller: pointer.BoolPtr(true), + Controller: ptr.To(true), }, }, Annotations: map[string]string{ @@ -222,7 +223,7 @@ func TestReconcilePersistentPodState(t *testing.T) { name: "kruise statefulset, scale down replicas 10->8, 1 pod deleted, 1 pod running", getSts: func() (*apps.StatefulSet, *appsv1beta1.StatefulSet) { kruise := kruiseStsDemo.DeepCopy() - kruise.Spec.Replicas = pointer.Int32Ptr(8) + kruise.Spec.Replicas = ptr.To[int32](8) return nil, kruise }, getPods: func() []*corev1.Pod { @@ -316,8 +317,12 @@ func TestReconcilePersistentPodState(t *testing.T) { name: "kruise reserveOrigin statefulset, scale down replicas 10->8, 1 pod deleted, 1 pod running", getSts: func() (*apps.StatefulSet, *appsv1beta1.StatefulSet) { kruise := kruiseStsDemo.DeepCopy() - kruise.Spec.Replicas = pointer.Int32Ptr(8) - kruise.Spec.ReserveOrdinals = []int{0, 3, 7} + kruise.Spec.Replicas = ptr.To[int32](8) + kruise.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(0), + intstr.FromInt32(3), + intstr.FromInt32(7), + } return nil, kruise }, getPods: func() []*corev1.Pod { diff --git a/pkg/controller/statefulset/stateful_pod_control_test.go b/pkg/controller/statefulset/stateful_pod_control_test.go index f20f3f8ece..902b5c8d78 100644 --- a/pkg/controller/statefulset/stateful_pod_control_test.go +++ b/pkg/controller/statefulset/stateful_pod_control_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" @@ -1045,7 +1046,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) { WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType, } - set.Spec.ReserveOrdinals = []int{2, 4} + set.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(2), + intstr.FromInt32(4), + } return set }, getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod { @@ -1083,7 +1087,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) { WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, } - set.Spec.ReserveOrdinals = []int{2, 4} + set.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(2), + intstr.FromInt32(4), + } return set }, getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod { @@ -1121,7 +1128,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) { WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, } - set.Spec.ReserveOrdinals = []int{2, 4} + set.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(2), + intstr.FromInt32(4), + } return set }, getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod { @@ -1170,7 +1180,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) { WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType, } - set.Spec.ReserveOrdinals = []int{2, 4} + set.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(2), + intstr.FromInt32(4), + } return set }, getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod { diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index cba0236c9d..d78bf3535d 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -4651,7 +4651,8 @@ func TestScaleUpWithMaxUnavailable(t *testing.T) { } func isOrHasInternalError(err error) bool { - agg, ok := err.(utilerrors.Aggregate) + var agg utilerrors.Aggregate + ok := errors.As(err, &agg) return !ok && !apierrors.IsInternalError(err) || ok && len(agg.Errors()) > 0 && !apierrors.IsInternalError(agg.Errors()[0]) } @@ -4662,10 +4663,12 @@ func emptyInvariants(set *appsv1beta1.StatefulSet, om *fakeObjectManager) error func TestStatefulSetControlWithStartOrdinal(t *testing.T) { defer utilfeature.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetStartOrdinal, true)() - simpleSetFn := func(replicas, startOrdinal int, reservedIds ...int) *appsv1beta1.StatefulSet { + simpleSetFn := func(replicas, startOrdinal int, reservedIds ...int32) *appsv1beta1.StatefulSet { statefulSet := newStatefulSet(replicas) statefulSet.Spec.Ordinals = &appsv1beta1.StatefulSetOrdinals{Start: int32(startOrdinal)} - statefulSet.Spec.ReserveOrdinals = append([]int{}, reservedIds...) + for _, id := range reservedIds { + statefulSet.Spec.ReserveOrdinals = append(statefulSet.Spec.ReserveOrdinals, intstr.FromInt32(id)) + } return statefulSet } diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index 174ab161b2..cace558df9 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -92,7 +92,7 @@ func podInOrdinalRange(pod *v1.Pod, set *appsv1beta1.StatefulSet) bool { return podInOrdinalRangeWithParams(pod, startOrdinal, endOrdinal, reserveOrdinals) } -func podInOrdinalRangeWithParams(pod *v1.Pod, startOrdinal, endOrdinal int, reserveOrdinals sets.Int) bool { +func podInOrdinalRangeWithParams(pod *v1.Pod, startOrdinal, endOrdinal int, reserveOrdinals sets.Set[int]) bool { ordinal := getOrdinal(pod) return ordinal >= startOrdinal && ordinal < endOrdinal && !reserveOrdinals.Has(ordinal) @@ -791,8 +791,8 @@ func decreaseAndCheckMaxUnavailable(maxUnavailable *int) bool { // result is startOrdinal 2(inclusive), endOrdinal 7(exclusive), reserveOrdinals = {1, 3} // replicas[endOrdinal - startOrdinal] stores [replica-2, nil(reserveOrdinal 3), replica-4, replica-5, replica-6] // todo: maybe we should remove ineffective reserveOrdinals in webhook, reserveOrdinals = {3} -func getStatefulSetReplicasRange(set *appsv1beta1.StatefulSet) (int, int, sets.Int) { - reserveOrdinals := sets.NewInt(set.Spec.ReserveOrdinals...) +func getStatefulSetReplicasRange(set *appsv1beta1.StatefulSet) (int, int, sets.Set[int]) { + reserveOrdinals := set.Spec.ReserveOrdinals.GetIntSet() replicaMaxOrdinal := getStartOrdinal(set) for realReplicaCount := 0; realReplicaCount < int(*set.Spec.Replicas); replicaMaxOrdinal++ { if reserveOrdinals.Has(replicaMaxOrdinal) { diff --git a/pkg/controller/statefulset/stateful_set_utils_test.go b/pkg/controller/statefulset/stateful_set_utils_test.go index 41ac1dddf4..e80b9909e9 100644 --- a/pkg/controller/statefulset/stateful_set_utils_test.go +++ b/pkg/controller/statefulset/stateful_set_utils_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller/history" @@ -913,8 +914,11 @@ func TestGetStatefulSetReplicasRange(t *testing.T) { name: "Ordinals start 0", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(4), - ReserveOrdinals: []int{1, 3}, + Replicas: int32Ptr(4), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromInt32(3), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -927,8 +931,11 @@ func TestGetStatefulSetReplicasRange(t *testing.T) { name: "Ordinals start 2 with ReserveOrdinals 1&3", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(4), - ReserveOrdinals: []int{1, 3}, + Replicas: int32Ptr(4), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromInt32(3), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -941,8 +948,11 @@ func TestGetStatefulSetReplicasRange(t *testing.T) { name: "Ordinals start 3 with ReserveOrdinals 1&3", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(4), - ReserveOrdinals: []int{1, 3}, + Replicas: int32Ptr(4), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromInt32(3), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 3, }, @@ -955,8 +965,11 @@ func TestGetStatefulSetReplicasRange(t *testing.T) { name: "Ordinals start 4 with ReserveOrdinals 1&3", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(4), - ReserveOrdinals: []int{1, 3}, + Replicas: int32Ptr(4), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromInt32(3), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 4, }, @@ -1034,7 +1047,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1065,7 +1078,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1096,7 +1109,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1130,7 +1143,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1164,7 +1177,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1197,7 +1210,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1230,8 +1243,10 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { name: "ReservedId 1, partition 2, create current pod1", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(3), - ReserveOrdinals: []int{1}, + Replicas: int32Ptr(3), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1267,7 +1282,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1298,7 +1313,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1329,7 +1344,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1363,7 +1378,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1397,7 +1412,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1430,7 +1445,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1465,7 +1480,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1504,7 +1519,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1545,7 +1560,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1584,7 +1599,7 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ Replicas: int32Ptr(3), - ReserveOrdinals: []int{}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{}, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, @@ -1624,8 +1639,10 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { name: "ReservedIds 1, UnorderedUpdate, partition 2 with update 1", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(3), - ReserveOrdinals: []int{1}, + Replicas: int32Ptr(3), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1665,8 +1682,10 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { name: "ReservedIds 1, UnorderedUpdate, partition 2 with update 1", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(3), - ReserveOrdinals: []int{1}, + Replicas: int32Ptr(3), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1711,8 +1730,10 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { name: "Ordinals start 0, reservedId 1, partition 1, create pod2", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(3), - ReserveOrdinals: []int{1}, + Replicas: int32Ptr(3), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 0, }, @@ -1746,8 +1767,10 @@ func TestIsCurrentRevisionNeeded(t *testing.T) { name: "Ordinals start 2, reservedId 1, partition 1, create pod2", statefulSet: &appsv1beta1.StatefulSet{ Spec: appsv1beta1.StatefulSetSpec{ - Replicas: int32Ptr(3), - ReserveOrdinals: []int{1}, + Replicas: int32Ptr(3), + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + }, Ordinals: &appsv1beta1.StatefulSetOrdinals{ Start: 2, }, diff --git a/pkg/util/controllerfinder/controller_finder.go b/pkg/util/controllerfinder/controller_finder.go index 6553a28875..91a266fd1d 100644 --- a/pkg/util/controllerfinder/controller_finder.go +++ b/pkg/util/controllerfinder/controller_finder.go @@ -77,7 +77,7 @@ type ScaleAndSelector struct { // controller.spec.Replicas; the value -1 means it is uncertain currently Scale int32 // kruise statefulSet.spec.ReserveOrdinals - ReserveOrdinals []int + ReserveOrdinals sets.Set[int] // controller.spec.Selector Selector *metav1.LabelSelector // metadata @@ -381,7 +381,7 @@ func (r *ControllerFinder) getPodKruiseStatefulSet(ref ControllerReference, name return &ScaleAndSelector{ Scale: *(ss.Spec.Replicas), - ReserveOrdinals: ss.Spec.ReserveOrdinals, + ReserveOrdinals: ss.Spec.ReserveOrdinals.GetIntSet(), Selector: ss.Spec.Selector, ControllerReference: ControllerReference{ APIVersion: ss.APIVersion, diff --git a/pkg/webhook/statefulset/validating/statefulset_validation.go b/pkg/webhook/statefulset/validating/statefulset_validation.go index 37103dce36..9c5d61a2a9 100644 --- a/pkg/webhook/statefulset/validating/statefulset_validation.go +++ b/pkg/webhook/statefulset/validating/statefulset_validation.go @@ -14,7 +14,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" @@ -44,15 +43,16 @@ func validatePodManagementPolicy(spec *appsv1beta1.StatefulSetSpec, fldPath *fie func validateReserveOrdinals(spec *appsv1beta1.StatefulSetSpec, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - - if spec.ReserveOrdinals != nil { - orders := sets.NewInt(spec.ReserveOrdinals...) - if orders.Len() != len(spec.ReserveOrdinals) { - allErrs = append(allErrs, field.Invalid(fldPath.Root(), spec.ReserveOrdinals, "reserveOrdinals contains duplicated items")) - } - for _, i := range spec.ReserveOrdinals { - if i < 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Root(), spec.ReserveOrdinals, fmt.Sprintf("reserveOrdinals contains %d which must be order >= 0", i))) + if len(spec.ReserveOrdinals) > 0 { + matcher := regexp.MustCompile(`^\d+-\d+$`) + for i, elem := range spec.ReserveOrdinals { + if elem.Type == intstr.String && !matcher.MatchString(elem.StrVal) { + allErrs = append(allErrs, field.Invalid(fldPath.Root(), spec.ReserveOrdinals, + fmt.Sprintf("%d th reserve ordinal is not a valid range: %s", i, elem.StrVal))) + } + if elem.Type == intstr.Int && elem.IntVal < 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Root(), spec.ReserveOrdinals, + fmt.Sprintf("%d th reserve ordinal is negative: %d", i, elem.IntVal))) } } } diff --git a/pkg/webhook/statefulset/validating/statefulset_validation_test.go b/pkg/webhook/statefulset/validating/statefulset_validation_test.go index b73305b3e0..bffce36546 100644 --- a/pkg/webhook/statefulset/validating/statefulset_validation_test.go +++ b/pkg/webhook/statefulset/validating/statefulset_validation_test.go @@ -545,7 +545,9 @@ func TestValidateStatefulSetUpdate(t *testing.T) { Spec: appsv1beta1.StatefulSetSpec{ Replicas: utilpointer.Int32Ptr(5), RevisionHistoryLimit: utilpointer.Int32Ptr(5), - ReserveOrdinals: []int{1}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + }, Lifecycle: &appspub.Lifecycle{PreDelete: &appspub.LifecycleHook{FinalizersHandler: []string{"foo/bar"}}}, Template: validPodTemplate1.Template, VolumeClaimTemplates: []v1.PersistentVolumeClaim{validVolumeClaimTemplate("30Gi")}, @@ -569,7 +571,9 @@ func TestValidateStatefulSetUpdate(t *testing.T) { Spec: appsv1beta1.StatefulSetSpec{ Replicas: utilpointer.Int32Ptr(10), RevisionHistoryLimit: utilpointer.Int32Ptr(10), - ReserveOrdinals: []int{2}, + ReserveOrdinals: appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(2), + }, Lifecycle: &appspub.Lifecycle{PreDelete: &appspub.LifecycleHook{FinalizersHandler: []string{"foo/hello"}}}, Template: validPodTemplate2.Template, VolumeClaimTemplates: []v1.PersistentVolumeClaim{validVolumeClaimTemplate("60Gi")}, diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index d07a336027..f9519687dd 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -2293,7 +2293,10 @@ var _ = SIGDescribe("StatefulSet", func() { ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{ WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, } - ss.Spec.ReserveOrdinals = []int{0, 1} + ss.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(0), + intstr.FromInt32(1), + } _, err := kc.AppsV1beta1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) framework.ExpectNoError(err) @@ -2310,6 +2313,35 @@ var _ = SIGDescribe("StatefulSet", func() { framework.ExpectNoError(err) }) + ginkgo.It("should delete PVCs with a OnScaledown policy and range reserveOrdinals=[0,2-5]", func() { + if framework.SkipIfNoDefaultStorageClass(c) { + return + } + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, + } + ss.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(0), + intstr.FromString("2-5"), + } + _, err := kc.AppsV1beta1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist") + err = verifyStatefulSetPVCsExist(c, ss, []int{1, 6, 7}) + framework.ExpectNoError(err) + + ginkgo.By("Scaling stateful set " + ss.Name + " to one replica") + ss, err = framework.NewStatefulSetTester(c, kc).Scale(ss, 1) + framework.ExpectNoError(err) + + ginkgo.By("Verifying all but one PVC deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{1}) + framework.ExpectNoError(err) + }) + ginkgo.It("should delete PVCs after adopting pod (WhenDeleted)", func() { if framework.SkipIfNoDefaultStorageClass(c) { return From fa91c124b82646d4083941195ac1f84d9e1e3fc6 Mon Sep 17 00:00:00 2001 From: AiRanthem Date: Thu, 2 Jan 2025 20:05:02 +0800 Subject: [PATCH 2/4] feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem --- apis/apps/v1beta1/statefulset_types_test.go | 99 +++++++++++++++++++ .../statefulset/stateful_pod_control_test.go | 53 +++++++++- .../statefulset/stateful_set_control_test.go | 1 - .../stateful_set_status_updater_test.go | 1 - .../statefulset/stateful_set_utils_test.go | 1 - .../statefulset_controller_test.go | 1 - 6 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 apis/apps/v1beta1/statefulset_types_test.go diff --git a/apis/apps/v1beta1/statefulset_types_test.go b/apis/apps/v1beta1/statefulset_types_test.go new file mode 100644 index 0000000000..9837d6b75f --- /dev/null +++ b/apis/apps/v1beta1/statefulset_types_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2025 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestGetIntSet(t *testing.T) { + tests := []struct { + name string + input ReserveOrdinal + expected sets.Set[int] + wantErr bool + }{ + { + name: "integer elements", + input: ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromInt32(2), + intstr.FromInt32(3), + }, + expected: sets.New(1, 2, 3), + }, + { + name: "string range", + input: ReserveOrdinal{ + intstr.FromString("1-3"), + }, + expected: sets.New(1, 2, 3), + }, + { + name: "invalid range end", + input: ReserveOrdinal{ + intstr.FromString("1-%"), + }, + wantErr: true, + }, + { + name: "invalid range start", + input: ReserveOrdinal{ + intstr.FromString("%-2"), + }, + wantErr: true, + }, + { + name: "invalid range split", + input: ReserveOrdinal{ + intstr.FromString("1-2-3"), + }, + wantErr: true, + }, + { + name: "mixed input", + input: ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromString("2-3"), + }, + expected: sets.New(1, 2, 3), + }, + { + name: "empty input", + input: ReserveOrdinal{}, + expected: sets.New[int](), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := tt.input.GetIntSet() + if tt.wantErr { + if actual != nil { + t.Errorf("Expected error (nil set), but got %v", actual) + } + return + } + if !actual.Equal(tt.expected) { + t.Errorf("For case %q, expected set %v, but got %v", tt.name, tt.expected, actual) + } + }) + } +} diff --git a/pkg/controller/statefulset/stateful_pod_control_test.go b/pkg/controller/statefulset/stateful_pod_control_test.go index 902b5c8d78..e035123674 100644 --- a/pkg/controller/statefulset/stateful_pod_control_test.go +++ b/pkg/controller/statefulset/stateful_pod_control_test.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kruise Authors. -Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1224,6 +1223,58 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) { } }, }, + { + name: "reserveOrdinals is [1,3-5], scaleDown=true, whenScaled=Retain, whenDeleted=Delete", + getStatefulSet: func() *appsv1beta1.StatefulSet { + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType, + } + set.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ + intstr.FromInt32(1), + intstr.FromString("3-5"), + } + return set + }, + getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod { + setClone := set.DeepCopy() + setClone.Spec.Replicas = utilpointer.Int32(5) + startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(setClone) + pods := make([]*v1.Pod, 0) + expectIndex := []int{0, 2, 6, 7, 8} + currentIndex := make([]int, 0) + for i := startOrdinal; i < endOrdinal; i++ { + if reserveOrdinals.Has(i) { + continue + } + currentIndex = append(currentIndex, i) + pods = append(pods, newStatefulSetPod(set, i)) + } + if !reflect.DeepEqual(expectIndex, currentIndex) { + t.Fatalf("expect(%v), but get(%v)", expectIndex, currentIndex) + } + return pods + }, + expectPvcOwnerRef: func(pvcName string) metav1.OwnerReference { + sIndex1 := strings.Index(pvcName, "-") + 1 + podName := pvcName[sIndex1:] + sIndex2 := strings.LastIndex(pvcName, "-") + 1 + index, _ := strconv.Atoi(pvcName[sIndex2:]) + if index < 9 { + return metav1.OwnerReference{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "foo", + } + } + return metav1.OwnerReference{ + APIVersion: "v1", + Kind: "Pod", + Name: podName, + } + }, + }, } for _, cs := range cases { diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index d78bf3535d..21cabc770f 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kruise Authors. -Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/controller/statefulset/stateful_set_status_updater_test.go b/pkg/controller/statefulset/stateful_set_status_updater_test.go index bbb6b30752..f225cd4864 100644 --- a/pkg/controller/statefulset/stateful_set_status_updater_test.go +++ b/pkg/controller/statefulset/stateful_set_status_updater_test.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kruise Authors. -Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/controller/statefulset/stateful_set_utils_test.go b/pkg/controller/statefulset/stateful_set_utils_test.go index e80b9909e9..71fcbd9f20 100644 --- a/pkg/controller/statefulset/stateful_set_utils_test.go +++ b/pkg/controller/statefulset/stateful_set_utils_test.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kruise Authors. -Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/controller/statefulset/statefulset_controller_test.go b/pkg/controller/statefulset/statefulset_controller_test.go index fcc3188812..a0529b3a8e 100644 --- a/pkg/controller/statefulset/statefulset_controller_test.go +++ b/pkg/controller/statefulset/statefulset_controller_test.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kruise Authors. -Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From ad3821c41eb72c411a538c2527c621823ebb279b Mon Sep 17 00:00:00 2001 From: AiRanthem Date: Fri, 3 Jan 2025 10:14:46 +0800 Subject: [PATCH 3/4] feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem --- .../validating/statefulset_validation_test.go | 137 +++++++++++++----- 1 file changed, 97 insertions(+), 40 deletions(-) diff --git a/pkg/webhook/statefulset/validating/statefulset_validation_test.go b/pkg/webhook/statefulset/validating/statefulset_validation_test.go index bffce36546..2b10ee1a82 100644 --- a/pkg/webhook/statefulset/validating/statefulset_validation_test.go +++ b/pkg/webhook/statefulset/validating/statefulset_validation_test.go @@ -32,8 +32,10 @@ import ( k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" corev1 "k8s.io/kubernetes/pkg/apis/core/v1" utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/fake" appspub "github.com/openkruise/kruise/apis/apps/pub" @@ -87,7 +89,7 @@ func TestValidateStatefulSet(t *testing.T) { var val2 int32 = 2 var val3 int32 = 3 var minus1 int32 = -1 - maxUnavailable1 := intstr.FromInt(1) + maxUnavailable1 := intstr.FromInt32(1) maxUnavailable120Percent := intstr.FromString("120%") successCases := []appsv1beta1.StatefulSet{ { @@ -140,7 +142,7 @@ func TestValidateStatefulSet(t *testing.T) { Partition: &val2, PodUpdatePolicy: appsv1beta1.RecreatePodUpdateStrategyType, MaxUnavailable: &maxUnavailable1, - MinReadySeconds: utilpointer.Int32Ptr(10), + MinReadySeconds: ptr.To[int32](10), } }()}, }, @@ -381,7 +383,7 @@ func TestValidateStatefulSet(t *testing.T) { Partition: &minus1, PodUpdatePolicy: appsv1beta1.RecreatePodUpdateStrategyType, MaxUnavailable: &maxUnavailable1, - MinReadySeconds: utilpointer.Int32Ptr(1), + MinReadySeconds: ptr.To[int32](1), } }()}, }, @@ -398,7 +400,7 @@ func TestValidateStatefulSet(t *testing.T) { Partition: &minus1, PodUpdatePolicy: appsv1beta1.RecreatePodUpdateStrategyType, MaxUnavailable: &maxUnavailable1, - MinReadySeconds: utilpointer.Int32Ptr(-1), + MinReadySeconds: ptr.To[int32](-1), }, }, }, @@ -415,7 +417,7 @@ func TestValidateStatefulSet(t *testing.T) { Partition: &minus1, PodUpdatePolicy: appsv1beta1.RecreatePodUpdateStrategyType, MaxUnavailable: &maxUnavailable1, - MinReadySeconds: utilpointer.Int32Ptr(appsv1beta1.MaxMinReadySeconds + 1), + MinReadySeconds: ptr.To[int32](appsv1beta1.MaxMinReadySeconds + 1), }, }, }, @@ -461,27 +463,27 @@ func TestValidateStatefulSet(t *testing.T) { } for i := range errs { - field := errs[i].Field - if !strings.HasPrefix(field, "spec.template.") && - field != "metadata.name" && - field != "metadata.namespace" && - field != "spec.selector" && - field != "spec.template" && - field != "GCEPersistentDisk.ReadOnly" && - field != "spec.replicas" && - field != "spec.template.labels" && - field != "metadata.annotations" && - field != "metadata.labels" && - field != "status.replicas" && - field != "spec.updateStrategy" && - field != "spec.updateStrategy.rollingUpdate" && - field != "spec.updateStrategy.rollingUpdate.partition" && - field != "spec.updateStrategy.rollingUpdate.maxUnavailable" && - field != "spec.updateStrategy.rollingUpdate.minReadySeconds" && - field != "spec.updateStrategy.rollingUpdate.podUpdatePolicy" && - field != "spec.template.spec.readinessGates" && - field != "spec.podManagementPolicy" && - field != "spec.template.spec.activeDeadlineSeconds" { + f := errs[i].Field + if !strings.HasPrefix(f, "spec.template.") && + f != "metadata.name" && + f != "metadata.namespace" && + f != "spec.selector" && + f != "spec.template" && + f != "GCEPersistentDisk.ReadOnly" && + f != "spec.replicas" && + f != "spec.template.labels" && + f != "metadata.annotations" && + f != "metadata.labels" && + f != "status.replicas" && + f != "spec.updateStrategy" && + f != "spec.updateStrategy.rollingUpdate" && + f != "spec.updateStrategy.rollingUpdate.partition" && + f != "spec.updateStrategy.rollingUpdate.maxUnavailable" && + f != "spec.updateStrategy.rollingUpdate.minReadySeconds" && + f != "spec.updateStrategy.rollingUpdate.podUpdatePolicy" && + f != "spec.template.spec.readinessGates" && + f != "spec.podManagementPolicy" && + f != "spec.template.spec.activeDeadlineSeconds" { t.Errorf("%s: missing prefix for: %v", k, errs[i]) } } @@ -543,8 +545,8 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ResourceVersion: "1", }, Spec: appsv1beta1.StatefulSetSpec{ - Replicas: utilpointer.Int32Ptr(5), - RevisionHistoryLimit: utilpointer.Int32Ptr(5), + Replicas: ptr.To[int32](5), + RevisionHistoryLimit: ptr.To[int32](5), ReserveOrdinals: appsv1beta1.ReserveOrdinal{ intstr.FromInt32(1), }, @@ -554,7 +556,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ScaleStrategy: &appsv1beta1.StatefulSetScaleStrategy{MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}}, UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{ Type: apps.RollingUpdateStatefulSetStrategyType, - RollingUpdate: &appsv1beta1.RollingUpdateStatefulSetStrategy{Partition: utilpointer.Int32Ptr(5)}, + RollingUpdate: &appsv1beta1.RollingUpdateStatefulSetStrategy{Partition: ptr.To[int32](5)}, }, PersistentVolumeClaimRetentionPolicy: &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{ WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType, @@ -569,8 +571,8 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ResourceVersion: "1", }, Spec: appsv1beta1.StatefulSetSpec{ - Replicas: utilpointer.Int32Ptr(10), - RevisionHistoryLimit: utilpointer.Int32Ptr(10), + Replicas: ptr.To[int32](10), + RevisionHistoryLimit: ptr.To[int32](10), ReserveOrdinals: appsv1beta1.ReserveOrdinal{ intstr.FromInt32(2), }, @@ -580,7 +582,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ScaleStrategy: &appsv1beta1.StatefulSetScaleStrategy{MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 2}}, UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{ Type: apps.RollingUpdateStatefulSetStrategyType, - RollingUpdate: &appsv1beta1.RollingUpdateStatefulSetStrategy{Partition: utilpointer.Int32Ptr(10)}, + RollingUpdate: &appsv1beta1.RollingUpdateStatefulSetStrategy{Partition: ptr.To[int32](10)}, }, PersistentVolumeClaimRetentionPolicy: &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{ WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType, @@ -614,7 +616,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { PodManagementPolicy: "", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}, Template: validPodTemplate1.Template, - Replicas: utilpointer.Int32Ptr(1), + Replicas: ptr.To[int32](1), UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, @@ -628,7 +630,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { PodManagementPolicy: "", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "bar"}}, Template: validPodTemplate1.Template, - Replicas: utilpointer.Int32Ptr(1), + Replicas: ptr.To[int32](1), UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, @@ -645,7 +647,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ServiceName: "foo", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}, Template: validPodTemplate1.Template, - Replicas: utilpointer.Int32Ptr(1), + Replicas: ptr.To[int32](1), UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, @@ -660,7 +662,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ServiceName: "bar", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}, Template: validPodTemplate1.Template, - Replicas: utilpointer.Int32Ptr(1), + Replicas: ptr.To[int32](1), UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, @@ -677,7 +679,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ServiceName: "bar", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}, Template: validPodTemplate1.Template, - Replicas: utilpointer.Int32Ptr(1), + Replicas: ptr.To[int32](1), UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, @@ -692,7 +694,7 @@ func TestValidateStatefulSetUpdate(t *testing.T) { ServiceName: "bar", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}, Template: validPodTemplate1.Template, - Replicas: utilpointer.Int32Ptr(1), + Replicas: ptr.To[int32](1), UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, @@ -709,8 +711,8 @@ func TestValidateStatefulSetUpdate(t *testing.T) { } for i := range errs { - field := errs[i].Field - if field != "spec" { + f := errs[i].Field + if f != "spec" { t.Errorf("%s: missing prefix for: %v", k, errs[i]) } } @@ -1136,3 +1138,58 @@ func TestGetDefaultStorageClass(t *testing.T) { } } + +func TestValidateReserveOrdinals(t *testing.T) { + tests := []struct { + name string + reserveOrdinals []intstr.IntOrString + expectedErrors bool + }{ + { + name: "EmptyReserveOrdinals", + reserveOrdinals: []intstr.IntOrString{}, + expectedErrors: false, + }, + { + name: "ValidStringRange", + reserveOrdinals: []intstr.IntOrString{ + {Type: intstr.String, StrVal: "1-2"}, + }, + expectedErrors: false, + }, + { + name: "InvalidStringRange", + reserveOrdinals: []intstr.IntOrString{ + {Type: intstr.String, StrVal: "12"}, + }, + expectedErrors: true, + }, + { + name: "ValidIntRange", + reserveOrdinals: []intstr.IntOrString{ + {Type: intstr.Int, IntVal: 1}, + }, + expectedErrors: false, + }, + { + name: "NegativeIntRange", + reserveOrdinals: []intstr.IntOrString{ + {Type: intstr.Int, IntVal: -1}, + }, + expectedErrors: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + spec := &appsv1beta1.StatefulSetSpec{ + ReserveOrdinals: test.reserveOrdinals, + } + fldPath := field.NewPath("spec") + errs := validateReserveOrdinals(spec, fldPath) + if len(errs) > 0 != test.expectedErrors { + t.Errorf("validateReserveOrdinals(%v) = %v, want %v", test.reserveOrdinals, errs, test.expectedErrors) + } + }) + } +} From e60e0a5441cdd3ea0b02359ab41cb11ae2a53ade Mon Sep 17 00:00:00 2001 From: AiRanthem Date: Fri, 3 Jan 2025 10:27:03 +0800 Subject: [PATCH 4/4] feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem --- test/e2e/apps/statefulset.go | 293 +++-------------------------------- 1 file changed, 19 insertions(+), 274 deletions(-) diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index f9519687dd..3e3070741d 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -24,7 +24,6 @@ import ( "reflect" "regexp" "strconv" - "strings" "time" "github.com/google/go-cmp/cmp" @@ -52,18 +51,6 @@ import ( "github.com/openkruise/kruise/test/e2e/framework" ) -const ( - zookeeperManifestPath = "test/e2e/testing-manifests/statefulset/zookeeper" - mysqlGaleraManifestPath = "test/e2e/testing-manifests/statefulset/mysql-galera" - redisManifestPath = "test/e2e/testing-manifests/statefulset/redis" - cockroachDBManifestPath = "test/e2e/testing-manifests/statefulset/cockroachdb" - // We don't restart MySQL cluster regardless of restartCluster, since MySQL doesn't handle restart well - restartCluster = true - - // Timeout for reads from databases running on stateful pods. - readTimeout = 60 * time.Second -) - // GCE Quota requirements: 3 pds, one per stateful pod manifest declared above. // GCE Api requirements: nodes and master need storage r/w permissions. var _ = SIGDescribe("AppStatefulSetStorage", func() { @@ -1770,14 +1757,14 @@ var _ = SIGDescribe("StatefulSet", func() { sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) ginkgo.By("Confirming that stateful set scale up will halt with unhealthy stateful pod") - sst.BreakHTTPProbe(ss) + _ = sst.BreakHTTPProbe(ss) sst.WaitForRunningAndNotReady(*ss.Spec.Replicas, ss) sst.WaitForStatusReadyReplicas(ss, 0) sst.UpdateReplicas(ss, 3) sst.ConfirmStatefulPodCount(1, ss, 10*time.Second, true) ginkgo.By("Scaling up stateful set " + ssName + " to 3 replicas and waiting until all of them will be running in namespace " + ns) - sst.RestoreHTTPProbe(ss) + _ = sst.RestoreHTTPProbe(ss) sst.WaitForRunningAndReady(3, ss) ginkgo.By("Verifying that stateful set " + ssName + " was scaled up in order") @@ -1803,15 +1790,15 @@ var _ = SIGDescribe("StatefulSet", func() { }) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - sst.BreakHTTPProbe(ss) + _ = sst.BreakHTTPProbe(ss) sst.WaitForStatusReadyReplicas(ss, 0) sst.WaitForRunningAndNotReady(3, ss) sst.UpdateReplicas(ss, 0) sst.ConfirmStatefulPodCount(3, ss, 10*time.Second, true) ginkgo.By("Scaling down stateful set " + ssName + " to 0 replicas and waiting until none of pods will run in namespace" + ns) - sst.RestoreHTTPProbe(ss) - sst.Scale(ss, 0) + _ = sst.RestoreHTTPProbe(ss) + _, _ = sst.Scale(ss, 0) ginkgo.By("Verifying that stateful set " + ssName + " was scaled down in reverse order") expectedOrder = []string{ssName + "-2", ssName + "-1", ssName + "-0"} @@ -1851,26 +1838,26 @@ var _ = SIGDescribe("StatefulSet", func() { sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) ginkgo.By("Confirming that stateful set scale up will not halt with unhealthy stateful pod") - sst.BreakHTTPProbe(ss) + _ = sst.BreakHTTPProbe(ss) sst.WaitForRunningAndNotReady(*ss.Spec.Replicas, ss) sst.WaitForStatusReadyReplicas(ss, 0) sst.UpdateReplicas(ss, 3) sst.ConfirmStatefulPodCount(3, ss, 10*time.Second, false) ginkgo.By("Scaling up stateful set " + ssName + " to 3 replicas and waiting until all of them will be running in namespace " + ns) - sst.RestoreHTTPProbe(ss) + _ = sst.RestoreHTTPProbe(ss) sst.WaitForRunningAndReady(3, ss) ginkgo.By("Scale down will not halt with unhealthy stateful pod") - sst.BreakHTTPProbe(ss) + _ = sst.BreakHTTPProbe(ss) sst.WaitForStatusReadyReplicas(ss, 0) sst.WaitForRunningAndNotReady(3, ss) sst.UpdateReplicas(ss, 0) sst.ConfirmStatefulPodCount(0, ss, 10*time.Second, false) ginkgo.By("Scaling down stateful set " + ssName + " to 0 replicas and waiting until none of pods will run in namespace" + ns) - sst.RestoreHTTPProbe(ss) - sst.Scale(ss, 0) + _ = sst.RestoreHTTPProbe(ss) + _, _ = sst.Scale(ss, 0) sst.WaitForStatusReplicas(ss, 0) }) @@ -2016,7 +2003,7 @@ var _ = SIGDescribe("StatefulSet", func() { */ framework.ConformanceIt("Should can update pods when the statefulset scale strategy is set", func() { ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) - maxUnavailable := intstr.FromInt(2) + maxUnavailable := intstr.FromInt32(2) ss := framework.NewStatefulSet(ssName, ns, headlessSvcName, 3, nil, nil, labels) ss.Spec.Template.Spec.Containers[0].Name = "busybox" ss.Spec.Template.Spec.Containers[0].Image = BusyboxImage @@ -2284,35 +2271,6 @@ var _ = SIGDescribe("StatefulSet", func() { framework.ExpectNoError(err) }) - ginkgo.It("should delete PVCs with a OnScaledown policy and reserveOrdinals=[0,1]", func() { - if framework.SkipIfNoDefaultStorageClass(c) { - return - } - ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) - *(ss.Spec.Replicas) = 3 - ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{ - WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType, - } - ss.Spec.ReserveOrdinals = appsv1beta1.ReserveOrdinal{ - intstr.FromInt32(0), - intstr.FromInt32(1), - } - _, err := kc.AppsV1beta1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - ginkgo.By("Confirming all 3 PVCs exist") - err = verifyStatefulSetPVCsExist(c, ss, []int{2, 3, 4}) - framework.ExpectNoError(err) - - ginkgo.By("Scaling stateful set " + ss.Name + " to one replica") - ss, err = framework.NewStatefulSetTester(c, kc).Scale(ss, 1) - framework.ExpectNoError(err) - - ginkgo.By("Verifying all but one PVC deleted") - err = verifyStatefulSetPVCsExist(c, ss, []int{2}) - framework.ExpectNoError(err) - }) - ginkgo.It("should delete PVCs with a OnScaledown policy and range reserveOrdinals=[0,2-5]", func() { if framework.SkipIfNoDefaultStorageClass(c) { return @@ -2689,219 +2647,6 @@ var _ = SIGDescribe("StatefulSet", func() { }) }) -func kubectlExecWithRetries(args ...string) (out string) { - var err error - for i := 0; i < 3; i++ { - if out, err = framework.RunKubectl(args...); err == nil { - return - } - framework.Logf("Retrying %v:\nerror %v\nstdout %v", args, err, out) - } - framework.Failf("Failed to execute \"%v\" with retries: %v", args, err) - return -} - -type statefulPodTester interface { - deploy(ns string) *appsv1beta1.StatefulSet - write(statefulPodIndex int, kv map[string]string) - read(statefulPodIndex int, key string) string - name() string -} - -//type clusterAppTester struct { -// ns string -// statefulPod statefulPodTester -// tester *framework.StatefulSetTester -//} -// -//func (c *clusterAppTester) run() { -// ginkgo.By("Deploying " + c.statefulPod.name()) -// ss := c.statefulPod.deploy(c.ns) -// -// ginkgo.By("Creating foo:bar in member with index 0") -// c.statefulPod.write(0, map[string]string{"foo": "bar"}) -// -// switch c.statefulPod.(type) { -// case *mysqlGaleraTester: -// // Don't restart MySQL cluster since it doesn't handle restarts well -// default: -// if restartCluster { -// ginkgo.By("Restarting stateful set " + ss.Name) -// c.tester.Restart(ss) -// c.tester.WaitForRunningAndReady(*ss.Spec.Replicas, ss) -// } -// } -// -// ginkgo.By("Reading value under foo from member with index 2") -// if err := pollReadWithTimeout(c.statefulPod, 2, "foo", "bar"); err != nil { -// framework.Failf("%v", err) -// } -//} -// -//type zookeeperTester struct { -// ss *appsv1beta1.StatefulSet -// tester *framework.StatefulSetTester -//} -// -//func (z *zookeeperTester) name() string { -// return "zookeeper" -//} -// -//func (z *zookeeperTester) deploy(ns string) *appsv1beta1.StatefulSet { -// z.ss = z.tester.CreateStatefulSet(zookeeperManifestPath, ns) -// return z.ss -//} -// -//func (z *zookeeperTester) write(statefulPodIndex int, kv map[string]string) { -// name := fmt.Sprintf("%v-%d", z.ss.Name, statefulPodIndex) -// ns := fmt.Sprintf("--namespace=%v", z.ss.Namespace) -// for k, v := range kv { -// cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh create /%v %v", k, v) -// framework.Logf(framework.RunKubectlOrDie("exec", ns, name, "--", "/bin/sh", "-c", cmd)) -// } -//} -// -//func (z *zookeeperTester) read(statefulPodIndex int, key string) string { -// name := fmt.Sprintf("%v-%d", z.ss.Name, statefulPodIndex) -// ns := fmt.Sprintf("--namespace=%v", z.ss.Namespace) -// cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh get /%v", key) -// return lastLine(framework.RunKubectlOrDie("exec", ns, name, "--", "/bin/sh", "-c", cmd)) -//} -// -//type mysqlGaleraTester struct { -// ss *appsv1beta1.StatefulSet -// tester *framework.StatefulSetTester -//} -// -//func (m *mysqlGaleraTester) name() string { -// return "mysql: galera" -//} -// -//func (m *mysqlGaleraTester) mysqlExec(cmd, ns, podName string) string { -// cmd = fmt.Sprintf("/usr/bin/mysql -u root -B -e '%v'", cmd) -// // TODO: Find a readiness probe for mysql that guarantees writes will -// // succeed and ditch retries. Current probe only reads, so there's a window -// // for a race. -// return kubectlExecWithRetries(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd) -//} -// -//func (m *mysqlGaleraTester) deploy(ns string) *appsv1beta1.StatefulSet { -// m.ss = m.tester.CreateStatefulSet(mysqlGaleraManifestPath, ns) -// -// framework.Logf("Deployed statefulset %v, initializing database", m.ss.Name) -// for _, cmd := range []string{ -// "create database statefulset;", -// "use statefulset; create table foo (k varchar(20), v varchar(20));", -// } { -// framework.Logf(m.mysqlExec(cmd, ns, fmt.Sprintf("%v-0", m.ss.Name))) -// } -// return m.ss -//} -// -//func (m *mysqlGaleraTester) write(statefulPodIndex int, kv map[string]string) { -// name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex) -// for k, v := range kv { -// cmd := fmt.Sprintf("use statefulset; insert into foo (k, v) values (\"%v\", \"%v\");", k, v) -// framework.Logf(m.mysqlExec(cmd, m.ss.Namespace, name)) -// } -//} -// -//func (m *mysqlGaleraTester) read(statefulPodIndex int, key string) string { -// name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex) -// return lastLine(m.mysqlExec(fmt.Sprintf("use statefulset; select v from foo where k=\"%v\";", key), m.ss.Namespace, name)) -//} -// -//type redisTester struct { -// ss *appsv1beta1.StatefulSet -// tester *framework.StatefulSetTester -//} -// -//func (m *redisTester) name() string { -// return "redis: master/slave" -//} -// -//func (m *redisTester) redisExec(cmd, ns, podName string) string { -// cmd = fmt.Sprintf("/opt/redis/redis-cli -h %v %v", podName, cmd) -// return framework.RunKubectlOrDie(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd) -//} -// -//func (m *redisTester) deploy(ns string) *appsv1beta1.StatefulSet { -// m.ss = m.tester.CreateStatefulSet(redisManifestPath, ns) -// return m.ss -//} -// -//func (m *redisTester) write(statefulPodIndex int, kv map[string]string) { -// name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex) -// for k, v := range kv { -// framework.Logf(m.redisExec(fmt.Sprintf("SET %v %v", k, v), m.ss.Namespace, name)) -// } -//} -// -//func (m *redisTester) read(statefulPodIndex int, key string) string { -// name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex) -// return lastLine(m.redisExec(fmt.Sprintf("GET %v", key), m.ss.Namespace, name)) -//} -// -//type cockroachDBTester struct { -// ss *appsv1beta1.StatefulSet -// tester *framework.StatefulSetTester -//} -// -//func (c *cockroachDBTester) name() string { -// return "CockroachDB" -//} -// -//func (c *cockroachDBTester) cockroachDBExec(cmd, ns, podName string) string { -// cmd = fmt.Sprintf("/cockroach/cockroach sql --insecure --host %s.cockroachdb -e \"%v\"", podName, cmd) -// return framework.RunKubectlOrDie(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd) -//} -// -//func (c *cockroachDBTester) deploy(ns string) *appsv1beta1.StatefulSet { -// c.ss = c.tester.CreateStatefulSet(cockroachDBManifestPath, ns) -// framework.Logf("Deployed statefulset %v, initializing database", c.ss.Name) -// for _, cmd := range []string{ -// "CREATE DATABASE IF NOT EXISTS foo;", -// "CREATE TABLE IF NOT EXISTS foo.bar (k STRING PRIMARY KEY, v STRING);", -// } { -// framework.Logf(c.cockroachDBExec(cmd, ns, fmt.Sprintf("%v-0", c.ss.Name))) -// } -// return c.ss -//} -// -//func (c *cockroachDBTester) write(statefulPodIndex int, kv map[string]string) { -// name := fmt.Sprintf("%v-%d", c.ss.Name, statefulPodIndex) -// for k, v := range kv { -// cmd := fmt.Sprintf("UPSERT INTO foo.bar VALUES ('%v', '%v');", k, v) -// framework.Logf(c.cockroachDBExec(cmd, c.ss.Namespace, name)) -// } -//} -//func (c *cockroachDBTester) read(statefulPodIndex int, key string) string { -// name := fmt.Sprintf("%v-%d", c.ss.Name, statefulPodIndex) -// return lastLine(c.cockroachDBExec(fmt.Sprintf("SELECT v FROM foo.bar WHERE k='%v';", key), c.ss.Namespace, name)) -//} - -func lastLine(out string) string { - outLines := strings.Split(strings.Trim(out, "\n"), "\n") - return outLines[len(outLines)-1] -} - -func pollReadWithTimeout(statefulPod statefulPodTester, statefulPodNumber int, key, expectedVal string) error { - err := wait.PollImmediate(time.Second, readTimeout, func() (bool, error) { - val := statefulPod.read(statefulPodNumber, key) - if val == "" { - return false, nil - } else if val != expectedVal { - return false, fmt.Errorf("expected value %v, found %v", expectedVal, val) - } - return true, nil - }) - - if err == wait.ErrWaitTimeout { - return fmt.Errorf("timed out when trying to read value for key %v from stateful pod %d", key, statefulPodNumber) - } - return err -} - // This function is used by two tests to test StatefulSet rollbacks: one using // PVCs and one using no storage. func rollbackTest(c clientset.Interface, kc kruiseclientset.Interface, ns string, ss *appsv1beta1.StatefulSet) { @@ -2991,7 +2736,7 @@ func rollbackTest(c clientset.Interface, kc kruiseclientset.Interface, ns string ginkgo.By("Rolling back update in reverse ordinal order") pods = sst.GetPodList(ss) sst.SortStatefulPods(pods) - sst.RestorePodHTTPProbe(ss, &pods.Items[1]) + _ = sst.RestorePodHTTPProbe(ss, &pods.Items[1]) ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) ss, pods = sst.WaitForRollingUpdate(ss) gomega.Expect(ss.Status.CurrentRevision).To(gomega.Equal(priorRevision), @@ -3023,7 +2768,7 @@ func verifyStatefulSetPVCsExist(c clientset.Interface, ss *appsv1beta1.StatefulS for _, id := range claimIds { idSet[id] = struct{}{} } - return wait.PollImmediate(framework.StatefulSetPoll, framework.StatefulSetTimeout, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), framework.StatefulSetPoll, framework.StatefulSetTimeout, true, func(_ context.Context) (bool, error) { pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) if err != nil { framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err) @@ -3068,7 +2813,7 @@ func verifyStatefulSetPVCsExistWithOwnerRefs(c clientset.Interface, kc kruisecli if setUID == "" { framework.Failf("Statefulset %s missing UID", ss.Name) } - return wait.PollImmediate(framework.StatefulSetPoll, framework.StatefulSetTimeout, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), framework.StatefulSetPoll, framework.StatefulSetTimeout, true, func(_ context.Context) (bool, error) { pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) if err != nil { framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err) @@ -3143,7 +2888,7 @@ func uncordonNode(ctx context.Context, c clientset.Interface, oldData, newData [ // waitForStatus waits for the StatefulSetStatus's CurrentReplicas to be equal to expectedReplicas // The returned StatefulSet contains such a StatefulSetStatus -func waitForStatusCurrentReplicas(ctx context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet, expectedReplicas int32) *appsv1beta1.StatefulSet { +func waitForStatusCurrentReplicas(_ context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet, expectedReplicas int32) *appsv1beta1.StatefulSet { sst := framework.NewStatefulSetTester(c, kc) sst.WaitForState(set, func(set2 *appsv1beta1.StatefulSet, pods *v1.PodList) (bool, error) { if set2.Status.ObservedGeneration >= set.Generation && set2.Status.CurrentReplicas == expectedReplicas { @@ -3157,7 +2902,7 @@ func waitForStatusCurrentReplicas(ctx context.Context, c clientset.Interface, kc // waitForStatus waits for the StatefulSetStatus's ObservedGeneration to be greater than or equal to set's Generation. // The returned StatefulSet contains such a StatefulSetStatus -func waitForStatus(ctx context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { +func waitForStatus(_ context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { sst := framework.NewStatefulSetTester(c, kc) sst.WaitForState(set, func(set2 *appsv1beta1.StatefulSet, pods *v1.PodList) (bool, error) { if set2.Status.ObservedGeneration >= set.Generation { @@ -3170,7 +2915,7 @@ func waitForStatus(ctx context.Context, c clientset.Interface, kc kruiseclientse } // waitForPodNames waits for the StatefulSet's pods to match expected names. -func waitForPodNames(ctx context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet, expectedPodNames []string) { +func waitForPodNames(_ context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet, expectedPodNames []string) { sst := framework.NewStatefulSetTester(c, kc) sst.WaitForState(set, func(intSet *appsv1beta1.StatefulSet, pods *v1.PodList) (bool, error) { @@ -3215,7 +2960,7 @@ type updateStatefulSetFunc func(*appsv1beta1.StatefulSet) func updateStatefulSetWithRetries(ctx context.Context, kc kruiseclientset.Interface, namespace, name string, applyUpdate updateStatefulSetFunc) (statefulSet *appsv1beta1.StatefulSet, err error) { statefulSets := kc.AppsV1beta1().StatefulSets(namespace) var updateErr error - pollErr := wait.PollWithContext(ctx, 10*time.Millisecond, 1*time.Minute, func(ctx context.Context) (bool, error) { + pollErr := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 1*time.Minute, true, func(ctx context.Context) (bool, error) { if statefulSet, err = statefulSets.Get(ctx, name, metav1.GetOptions{}); err != nil { return false, err } @@ -3235,7 +2980,7 @@ func updateStatefulSetWithRetries(ctx context.Context, kc kruiseclientset.Interf } // waitForPVCCapacity waits for the StatefulSet's pods to match expected names. -func waitForPVCCapacity(ctx context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet, cmp func(resource.Quantity, resource.Quantity) bool) { +func waitForPVCCapacity(_ context.Context, c clientset.Interface, kc kruiseclientset.Interface, set *appsv1beta1.StatefulSet, cmp func(resource.Quantity, resource.Quantity) bool) { sst := framework.NewStatefulSetTester(c, kc) capacityMap := map[string]resource.Quantity{} for _, pvc := range set.Spec.VolumeClaimTemplates {