From b5481a8e69e71fa8a8e0fe9149ba067a924f0b32 Mon Sep 17 00:00:00 2001 From: ChrisLiu Date: Mon, 18 Dec 2023 22:38:38 +0800 Subject: [PATCH] feat: add autoUpdateStrategy & new Type OnlyNew Signed-off-by: ChrisLiu --- apis/v1alpha1/gameserverset_types.go | 15 ++ apis/v1alpha1/zz_generated.deepcopy.go | 20 +++ .../bases/game.kruise.io_gameserversets.yaml | 9 ++ .../gameserverset/gameserverset_controller.go | 10 ++ .../gameserverset/gameserverset_manager.go | 22 +++ .../gameserverset_manager_test.go | 135 ++++++++++++++++++ test/e2e/client/client.go | 3 + test/e2e/framework/framework.go | 60 +++++--- test/e2e/testcase/testcase.go | 20 ++- 9 files changed, 277 insertions(+), 17 deletions(-) diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go index ffa19d99..a8aa922c 100644 --- a/apis/v1alpha1/gameserverset_types.go +++ b/apis/v1alpha1/gameserverset_types.go @@ -90,8 +90,23 @@ type UpdateStrategy struct { // RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. // +optional RollingUpdate *RollingUpdateStatefulSetStrategy `json:"rollingUpdate,omitempty"` + // AutoUpdateStrategy means that the update process will be performed automatically without user intervention. + // +optional + AutoUpdateStrategy *AutoUpdateStrategy `json:"autoUpdateStrategy,omitempty"` +} + +type AutoUpdateStrategy struct { + //+kubebuilder:validation:Required + Type AutoUpdateStrategyType `json:"type"` } +type AutoUpdateStrategyType string + +const ( + // OnlyNewAutoUpdateStrategyType indicates exist GameServers will never be updated, new GameServers will be created in new template. + OnlyNewAutoUpdateStrategyType AutoUpdateStrategyType = "OnlyNew" +) + type RollingUpdateStatefulSetStrategy struct { // Partition indicates the ordinal at which the StatefulSet should be partitioned by default. // But if unorderedUpdate has been set: diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 5f93d076..89f2e2ba 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,21 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoUpdateStrategy) DeepCopyInto(out *AutoUpdateStrategy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoUpdateStrategy. +func (in *AutoUpdateStrategy) DeepCopy() *AutoUpdateStrategy { + if in == nil { + return nil + } + out := new(AutoUpdateStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GameServer) DeepCopyInto(out *GameServer) { *out = *in @@ -577,6 +592,11 @@ func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { *out = new(RollingUpdateStatefulSetStrategy) (*in).DeepCopyInto(*out) } + if in.AutoUpdateStrategy != nil { + in, out := &in.AutoUpdateStrategy, &out.AutoUpdateStrategy + *out = new(AutoUpdateStrategy) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. diff --git a/config/crd/bases/game.kruise.io_gameserversets.yaml b/config/crd/bases/game.kruise.io_gameserversets.yaml index 12d1cce5..87122c05 100644 --- a/config/crd/bases/game.kruise.io_gameserversets.yaml +++ b/config/crd/bases/game.kruise.io_gameserversets.yaml @@ -608,6 +608,15 @@ spec: type: array updateStrategy: properties: + autoUpdateStrategy: + description: AutoUpdateStrategy means that the update process + will be performed automatically without user intervention. + properties: + type: + type: string + required: + - type + type: object rollingUpdate: description: RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. diff --git a/pkg/controllers/gameserverset/gameserverset_controller.go b/pkg/controllers/gameserverset/gameserverset_controller.go index 26a12c1f..1588d5e5 100644 --- a/pkg/controllers/gameserverset/gameserverset_controller.go +++ b/pkg/controllers/gameserverset/gameserverset_controller.go @@ -228,6 +228,16 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques return reconcile.Result{}, nil } + // adjust partition when autoUpdate + if gsm.IsNeedToAdjustPartition() { + err = gsm.AdjustPartition() + if err != nil { + klog.Errorf("GameServerSet %s failed to adjust partition in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + // update workload if gsm.IsNeedToUpdateWorkload() { err = gsm.UpdateWorkload() diff --git a/pkg/controllers/gameserverset/gameserverset_manager.go b/pkg/controllers/gameserverset/gameserverset_manager.go index 7a6e17a3..e9b69909 100644 --- a/pkg/controllers/gameserverset/gameserverset_manager.go +++ b/pkg/controllers/gameserverset/gameserverset_manager.go @@ -49,6 +49,8 @@ type Control interface { SyncPodProbeMarker() error SyncGameServerReplicas() error GetReplicasAfterKilling() *int32 + IsNeedToAdjustPartition() bool + AdjustPartition() error } const ( @@ -77,6 +79,26 @@ func NewGameServerSetManager(gss *gameKruiseV1alpha1.GameServerSet, asts *kruise } } +func (manager *GameServerSetManager) AdjustPartition() error { + gss := manager.gameServerSet + if gss.Spec.UpdateStrategy.RollingUpdate == nil { + gss.Spec.UpdateStrategy.RollingUpdate = &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{} + } + gss.Spec.UpdateStrategy.RollingUpdate.Partition = gss.Spec.Replicas + return manager.client.Update(context.Background(), gss) +} + +func (manager *GameServerSetManager) IsNeedToAdjustPartition() bool { + gss := manager.gameServerSet + if gss.Spec.UpdateStrategy.AutoUpdateStrategy == nil { + return false + } + if gss.Spec.UpdateStrategy.AutoUpdateStrategy.Type == gameKruiseV1alpha1.OnlyNewAutoUpdateStrategyType && (gss.Spec.UpdateStrategy.RollingUpdate == nil || gss.Spec.UpdateStrategy.RollingUpdate.Partition == nil || *gss.Spec.Replicas != *gss.Spec.UpdateStrategy.RollingUpdate.Partition) { + return true + } + return false +} + func (manager *GameServerSetManager) GetReplicasAfterKilling() *int32 { gss := manager.gameServerSet asts := manager.asts diff --git a/pkg/controllers/gameserverset/gameserverset_manager_test.go b/pkg/controllers/gameserverset/gameserverset_manager_test.go index e1f1e7fb..89885471 100644 --- a/pkg/controllers/gameserverset/gameserverset_manager_test.go +++ b/pkg/controllers/gameserverset/gameserverset_manager_test.go @@ -997,3 +997,138 @@ func TestNumberToKill(t *testing.T) { } } } + +func TestIsNeedToAdjustPartition(t *testing.T) { + tests := []struct { + gss *gameKruiseV1alpha1.GameServerSet + result bool + }{ + // case 0 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{}, + }, + }, + result: false, + }, + + // case 1 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ + AutoUpdateStrategy: &gameKruiseV1alpha1.AutoUpdateStrategy{ + Type: gameKruiseV1alpha1.OnlyNewAutoUpdateStrategyType, + }, + }, + }, + }, + result: true, + }, + + // case 2 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ + RollingUpdate: &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{ + Partition: pointer.Int32(3), + PodUpdatePolicy: kruiseV1beta1.InPlaceIfPossiblePodUpdateStrategyType, + }, + AutoUpdateStrategy: &gameKruiseV1alpha1.AutoUpdateStrategy{ + Type: gameKruiseV1alpha1.OnlyNewAutoUpdateStrategyType, + }, + }, + }, + }, + result: false, + }, + + // case 3 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ + RollingUpdate: &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{ + Partition: pointer.Int32(2), + PodUpdatePolicy: kruiseV1beta1.InPlaceIfPossiblePodUpdateStrategyType, + }, + AutoUpdateStrategy: &gameKruiseV1alpha1.AutoUpdateStrategy{ + Type: gameKruiseV1alpha1.OnlyNewAutoUpdateStrategyType, + }, + }, + }, + }, + result: true, + }, + } + + for i, test := range tests { + manager := &GameServerSetManager{ + gameServerSet: test.gss, + } + actual := manager.IsNeedToAdjustPartition() + expect := test.result + if actual != expect { + t.Errorf("case %d: expect IsNeedToAdjustPartition is %v but actually %v", i, expect, actual) + } + } +} + +func TestAdjustPartition(t *testing.T) { + tests := []struct { + gss *gameKruiseV1alpha1.GameServerSet + }{ + // case 0 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case0", + }, + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32(3), + }, + }, + }, + + // case 1 + { + gss: &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "xxx", + Name: "case1", + }, + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ + RollingUpdate: &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{ + Partition: pointer.Int32(4), + }, + }, + }, + }, + }, + } + + for i, test := range tests { + objs := []client.Object{test.gss} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + manager := &GameServerSetManager{ + gameServerSet: test.gss, + client: c, + } + if err := manager.AdjustPartition(); err != nil { + t.Error(err) + } + + if *manager.gameServerSet.Spec.Replicas != *manager.gameServerSet.Spec.UpdateStrategy.RollingUpdate.Partition { + t.Errorf("case %d: Replicas is not equal with Partition", i) + } + } +} diff --git a/test/e2e/client/client.go b/test/e2e/client/client.go index 24de22d8..f146acb3 100644 --- a/test/e2e/client/client.go +++ b/test/e2e/client/client.go @@ -11,6 +11,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" @@ -60,6 +61,7 @@ func (client *Client) DeleteNamespace() error { } func (client *Client) DefaultGameServerSet() *gameKruiseV1alpha1.GameServerSet { + maxUnavailable := intstr.FromString("100%") return &gameKruiseV1alpha1.GameServerSet{ ObjectMeta: metav1.ObjectMeta{ Name: GameServerSet, @@ -70,6 +72,7 @@ func (client *Client) DefaultGameServerSet() *gameKruiseV1alpha1.GameServerSet { UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ Type: apps.RollingUpdateStatefulSetStrategyType, RollingUpdate: &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{ + MaxUnavailable: &maxUnavailable, PodUpdatePolicy: kruiseV1beta1.InPlaceIfPossiblePodUpdateStrategyType, }, }, diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index efd6dd7d..6e09783c 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -110,6 +110,33 @@ func (f *Framework) GameServerScale(gss *gamekruiseiov1alpha1.GameServerSet, des return f.client.PatchGameServerSet(data) } +func (f *Framework) AutoUpdateOnlyNew() (*gamekruiseiov1alpha1.GameServerSet, error) { + conJson := map[string]interface{}{"spec": map[string]interface{}{"updateStrategy": map[string]interface{}{"autoUpdateStrategy": map[string]string{"type": "OnlyNew"}}}} + data, err := json.Marshal(conJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServerSet(data) +} + +func (f *Framework) WaitOnlyNewTakeEffect() error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + gss, err := f.client.GetGameServerSet() + if err != nil { + return false, err + } + + if gss.Spec.UpdateStrategy.RollingUpdate.Partition == nil { + return false, nil + } + if *gss.Spec.Replicas != *gss.Spec.UpdateStrategy.RollingUpdate.Partition { + return false, nil + } + return true, nil + }) +} + func (f *Framework) ImageUpdate(gss *gamekruiseiov1alpha1.GameServerSet, name, image string) (*gamekruiseiov1alpha1.GameServerSet, error) { var newContainers []corev1.Container for _, c := range gss.Spec.GameServerTemplate.Spec.Containers { @@ -164,7 +191,7 @@ func (f *Framework) WaitForGsCreated(gss *gamekruiseiov1alpha1.GameServerSet) er }) } -func (f *Framework) WaitForUpdated(gss *gamekruiseiov1alpha1.GameServerSet, name, image string) error { +func (f *Framework) WaitForUpdated(gss *gamekruiseiov1alpha1.GameServerSet, name, image string, updateIds []int) error { return wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) { gssName := gss.GetName() @@ -175,24 +202,25 @@ func (f *Framework) WaitForUpdated(gss *gamekruiseiov1alpha1.GameServerSet, name if err != nil { return false, err } - updated := 0 for _, pod := range podList.Items { - for _, c := range pod.Status.ContainerStatuses { - if name == c.Name && strings.Contains(c.Image, image) { - updated++ - break + id := util.GetIndexFromGsName(pod.GetName()) + if util.IsNumInList(id, updateIds) { + // should be updated + for _, c := range pod.Status.ContainerStatuses { + if name == c.Name && !strings.Contains(c.Image, image) { + return false, nil + } } - } - } - - if gss.Spec.UpdateStrategy.RollingUpdate == nil || gss.Spec.UpdateStrategy.RollingUpdate.Partition == nil { - if int32(updated) != *gss.Spec.Replicas { - return false, nil - } - } else { - if int32(updated) != *gss.Spec.Replicas-*gss.Spec.UpdateStrategy.RollingUpdate.Partition { - return false, nil + fmt.Printf("id %d updated. Passed.\n", id) + } else { + // should not be updated + for _, c := range pod.Status.ContainerStatuses { + if name == c.Name && strings.Contains(c.Image, image) { + return false, nil + } + } + fmt.Printf("id %d not updated. Passed.\n", id) } } return true, nil diff --git a/test/e2e/testcase/testcase.go b/test/e2e/testcase/testcase.go index 05a301b9..cac2be0d 100644 --- a/test/e2e/testcase/testcase.go +++ b/test/e2e/testcase/testcase.go @@ -71,7 +71,25 @@ func RunTestCases(f *framework.Framework) { gss, err = f.ImageUpdate(gss, client.GameContainerName, "nginx:latest") gomega.Expect(err).To(gomega.BeNil()) - err = f.WaitForUpdated(gss, client.GameContainerName, "nginx:latest") + err = f.WaitForUpdated(gss, client.GameContainerName, "nginx:latest", []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.AutoUpdateOnlyNew() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitOnlyNewTakeEffect() + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.ImageUpdate(gss, client.GameContainerName, "nginx:1.9.7") + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.GameServerScale(gss, 5, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2, 3, 4}) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForUpdated(gss, client.GameContainerName, "nginx:1.9.7", []int{3, 4}) gomega.Expect(err).To(gomega.BeNil()) })