diff --git a/.github/workflows/e2e-v1beta1-1.19.yaml b/.github/workflows/e2e-v1beta1-1.19.yaml new file mode 100644 index 00000000..6ec0e39d --- /dev/null +++ b/.github/workflows/e2e-v1beta1-1.19.yaml @@ -0,0 +1,110 @@ +name: E2E-V1Beta1-1.19 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.19' + KIND_IMAGE: 'kindest/node:v1.19.16' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='Step Jump' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal diff --git a/.github/workflows/e2e-v1beta1-1.23.yaml b/.github/workflows/e2e-v1beta1-1.23.yaml new file mode 100644 index 00000000..8dad2a8b --- /dev/null +++ b/.github/workflows/e2e-v1beta1-1.23.yaml @@ -0,0 +1,110 @@ +name: E2E-V1Beta1-1.23 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.19' + KIND_IMAGE: 'kindest/node:v1.23.3' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='Step Jump' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal diff --git a/api/v1alpha1/batchrelease_plan_types.go b/api/v1alpha1/batchrelease_plan_types.go index 0cf37e8c..7514b3cb 100644 --- a/api/v1alpha1/batchrelease_plan_types.go +++ b/api/v1alpha1/batchrelease_plan_types.go @@ -54,6 +54,8 @@ type ReleasePlan struct { // only support for canary deployment // +optional PatchPodTemplateMetadata *PatchPodTemplateMetadata `json:"patchPodTemplateMetadata,omitempty"` + // RollingStyle can be "Canary", "Partiton" or "BlueGreen" + RollingStyle RollingStyleType `json:"rollingStyle,omitempty"` } type FinalizingPolicyType string diff --git a/api/v1alpha1/conversion.go b/api/v1alpha1/conversion.go index 5164c1ca..6872c916 100644 --- a/api/v1alpha1/conversion.go +++ b/api/v1alpha1/conversion.go @@ -104,18 +104,22 @@ func (src *Rollout) ConvertTo(dst conversion.Hub) error { return nil } obj.Status.CanaryStatus = &v1beta1.CanaryStatus{ - ObservedWorkloadGeneration: src.Status.CanaryStatus.ObservedWorkloadGeneration, - ObservedRolloutID: src.Status.CanaryStatus.ObservedRolloutID, - RolloutHash: src.Status.CanaryStatus.RolloutHash, - StableRevision: src.Status.CanaryStatus.StableRevision, - CanaryRevision: src.Status.CanaryStatus.CanaryRevision, - PodTemplateHash: src.Status.CanaryStatus.PodTemplateHash, - CanaryReplicas: src.Status.CanaryStatus.CanaryReplicas, - CanaryReadyReplicas: src.Status.CanaryStatus.CanaryReadyReplicas, - CurrentStepIndex: src.Status.CanaryStatus.CurrentStepIndex, - CurrentStepState: v1beta1.CanaryStepState(src.Status.CanaryStatus.CurrentStepState), - Message: src.Status.CanaryStatus.Message, - LastUpdateTime: src.Status.CanaryStatus.LastUpdateTime, + CommonStatus: v1beta1.CommonStatus{ + ObservedWorkloadGeneration: src.Status.CanaryStatus.ObservedWorkloadGeneration, + ObservedRolloutID: src.Status.CanaryStatus.ObservedRolloutID, + RolloutHash: src.Status.CanaryStatus.RolloutHash, + StableRevision: src.Status.CanaryStatus.StableRevision, + PodTemplateHash: src.Status.CanaryStatus.PodTemplateHash, + CurrentStepIndex: src.Status.CanaryStatus.CurrentStepIndex, + CurrentStepState: v1beta1.CanaryStepState(src.Status.CanaryStatus.CurrentStepState), + Message: src.Status.CanaryStatus.Message, + LastUpdateTime: src.Status.CanaryStatus.LastUpdateTime, + FinalisingStep: v1beta1.FinalisingStepType(src.Status.CanaryStatus.FinalisingStep), + NextStepIndex: src.Status.CanaryStatus.NextStepIndex, + }, + CanaryRevision: src.Status.CanaryStatus.CanaryRevision, + CanaryReplicas: src.Status.CanaryStatus.CanaryReplicas, + CanaryReadyReplicas: src.Status.CanaryStatus.CanaryReadyReplicas, } return nil default: @@ -167,7 +171,9 @@ func (dst *Rollout) ConvertFrom(src conversion.Hub) error { case *v1beta1.Rollout: srcV1beta1 := src.(*v1beta1.Rollout) dst.ObjectMeta = srcV1beta1.ObjectMeta - + if !srcV1beta1.Spec.Strategy.IsCanaryStragegy() { + return fmt.Errorf("v1beta1 Rollout with %s strategy cannot be converted to v1alpha1", srcV1beta1.Spec.Strategy.GetRollingStyle()) + } // spec dst.Spec = RolloutSpec{ ObjectRef: ObjectRef{ @@ -254,6 +260,8 @@ func (dst *Rollout) ConvertFrom(src conversion.Hub) error { CurrentStepState: CanaryStepState(srcV1beta1.Status.CanaryStatus.CurrentStepState), Message: srcV1beta1.Status.CanaryStatus.Message, LastUpdateTime: srcV1beta1.Status.CanaryStatus.LastUpdateTime, + FinalisingStep: FinalizeStateType(srcV1beta1.Status.CanaryStatus.FinalisingStep), + NextStepIndex: srcV1beta1.Status.CanaryStatus.NextStepIndex, } return nil default: @@ -338,8 +346,15 @@ func (src *BatchRelease) ConvertTo(dst conversion.Hub) error { obj.Spec.ReleasePlan.PatchPodTemplateMetadata.Labels[k] = v } } - if !strings.EqualFold(src.Annotations[RolloutStyleAnnotation], string(PartitionRollingStyle)) { - obj.Spec.ReleasePlan.EnableExtraWorkloadForCanary = true + + if strings.EqualFold(src.Annotations[RolloutStyleAnnotation], string(PartitionRollingStyle)) { + obj.Spec.ReleasePlan.RollingStyle = v1beta1.PartitionRollingStyle + } + if strings.EqualFold(src.Annotations[RolloutStyleAnnotation], string(CanaryRollingStyle)) { + obj.Spec.ReleasePlan.RollingStyle = v1beta1.CanaryRollingStyle + } + if strings.EqualFold(src.Annotations[RolloutStyleAnnotation], string(BlueGreenRollingStyle)) { + obj.Spec.ReleasePlan.RollingStyle = v1beta1.BlueGreenRollingStyle } // status @@ -417,11 +432,8 @@ func (dst *BatchRelease) ConvertFrom(src conversion.Hub) error { if dst.Annotations == nil { dst.Annotations = map[string]string{} } - if srcV1beta1.Spec.ReleasePlan.EnableExtraWorkloadForCanary { - dst.Annotations[RolloutStyleAnnotation] = strings.ToLower(string(CanaryRollingStyle)) - } else { - dst.Annotations[RolloutStyleAnnotation] = strings.ToLower(string(PartitionRollingStyle)) - } + dst.Annotations[RolloutStyleAnnotation] = strings.ToLower(string(srcV1beta1.Spec.ReleasePlan.RollingStyle)) + dst.Spec.ReleasePlan.RollingStyle = RollingStyleType(srcV1beta1.Spec.ReleasePlan.RollingStyle) // status dst.Status = BatchReleaseStatus{ diff --git a/api/v1alpha1/deployment_types.go b/api/v1alpha1/deployment_types.go index 35202e5b..98fc474c 100644 --- a/api/v1alpha1/deployment_types.go +++ b/api/v1alpha1/deployment_types.go @@ -59,6 +59,8 @@ const ( PartitionRollingStyle RollingStyleType = "Partition" // CanaryRollingStyle means rolling in canary way, and will create a canary Deployment. CanaryRollingStyle RollingStyleType = "Canary" + // BlueGreenRollingStyle means rolling in blue-green way, and will NOT create a canary Deployment. + BlueGreenRollingStyle RollingStyleType = "BlueGreen" ) // DeploymentExtraStatus is extra status field for Advanced Deployment @@ -74,7 +76,7 @@ type DeploymentExtraStatus struct { } func SetDefaultDeploymentStrategy(strategy *DeploymentStrategy) { - if strategy.RollingStyle == CanaryRollingStyle { + if strategy.RollingStyle != PartitionRollingStyle { return } if strategy.RollingUpdate == nil { diff --git a/api/v1alpha1/rollout_types.go b/api/v1alpha1/rollout_types.go index 697287d8..df3fa933 100644 --- a/api/v1alpha1/rollout_types.go +++ b/api/v1alpha1/rollout_types.go @@ -242,16 +242,24 @@ type CanaryStatus struct { CanaryReplicas int32 `json:"canaryReplicas"` // CanaryReadyReplicas the numbers of ready canary revision pods CanaryReadyReplicas int32 `json:"canaryReadyReplicas"` - // CurrentStepIndex defines the current step of the rollout is on. If the current step index is null, the - // controller will execute the rollout. + // NextStepIndex defines the next step of the rollout is on. + // In normal case, NextStepIndex is equal to CurrentStepIndex + 1 + // If the current step is the last step, NextStepIndex is equal to 0 + // Before the release, NextStepIndex is also equal to 0 + // It is allowed to modify NextStepIndex by design, + // e.g. if CurrentStepIndex is 2, user can patch NextStepIndex to 3 (if exists) to + // achieve batch jump, or patch NextStepIndex to 1 to implement a re-execution of step 1 + NextStepIndex int32 `json:"nextStepIndex"` // +optional - CurrentStepIndex int32 `json:"currentStepIndex"` - CurrentStepState CanaryStepState `json:"currentStepState"` - Message string `json:"message,omitempty"` - LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + CurrentStepIndex int32 `json:"currentStepIndex"` + CurrentStepState CanaryStepState `json:"currentStepState"` + Message string `json:"message,omitempty"` + LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + FinalisingStep FinalizeStateType `json:"finalisingStep"` } type CanaryStepState string +type FinalizeStateType string const ( CanaryStepStateUpgrade CanaryStepState = "StepUpgrade" diff --git a/api/v1beta1/batchrelease_plan_types.go b/api/v1beta1/batchrelease_plan_types.go index 8b3c21a2..bf1940be 100644 --- a/api/v1beta1/batchrelease_plan_types.go +++ b/api/v1beta1/batchrelease_plan_types.go @@ -54,10 +54,8 @@ type ReleasePlan struct { // only support for canary deployment // +optional PatchPodTemplateMetadata *PatchPodTemplateMetadata `json:"patchPodTemplateMetadata,omitempty"` - // If true, then it will create new deployment for canary, such as: workload-demo-canary. - // When user verifies that the canary version is ready, we will remove the canary deployment and release the deployment workload-demo in full. - // Current only support k8s native deployment - EnableExtraWorkloadForCanary bool `json:"enableExtraWorkloadForCanary"` + // RollingStyle can be "Canary", "Partiton" or "BlueGreen" + RollingStyle RollingStyleType `json:"rollingStyle,omitempty"` } type FinalizingPolicyType string diff --git a/api/v1beta1/deployment_types.go b/api/v1beta1/deployment_types.go index 3db56143..9975e989 100644 --- a/api/v1beta1/deployment_types.go +++ b/api/v1beta1/deployment_types.go @@ -37,6 +37,16 @@ const ( // AdvancedDeploymentControlLabel is label for deployment, // which labels whether the deployment is controlled by advanced-deployment-controller. AdvancedDeploymentControlLabel = "rollouts.kruise.io/controlled-by-advanced-deployment-controller" + + // OriginalDeploymentStrategyAnnotation is annotation for workload in BlueGreen Release, + // it will store the original setting of the workload, which will be used to restore the workload + OriginalDeploymentStrategyAnnotation = "rollouts.kruise.io/original-deployment-strategy" + + // MaxProgressSeconds is the value we set for ProgressDeadlineSeconds + // MaxReadySeconds is the value we set for MinReadySeconds, which is one less than ProgressDeadlineSeconds + // MaxInt32: 2147483647, ≈ 68 years + MaxProgressSeconds = 1<<31 - 1 + MaxReadySeconds = MaxProgressSeconds - 1 ) // DeploymentStrategy is strategy field for Advanced Deployment @@ -52,6 +62,31 @@ type DeploymentStrategy struct { Partition intstr.IntOrString `json:"partition,omitempty"` } +// OriginalDeploymentStrategy stores part of the fileds of a workload, +// so that it can be restored when finalizing. +// It is only used for BlueGreen Release +// Similar to DeploymentStrategy, it is an annotation used in workload +// However, unlike DeploymentStrategy, it is only used to store and restore the user's strategy +type OriginalDeploymentStrategy struct { + // The deployment strategy to use to replace existing pods with new ones. + // +optional + // +patchStrategy=retainKeys + Strategy *apps.DeploymentStrategy `json:"strategy,omitempty" patchStrategy:"retainKeys" protobuf:"bytes,4,opt,name=strategy"` + + // Minimum number of seconds for which a newly created pod should be ready + // without any of its container crashing, for it to be considered available. + // Defaults to 0 (pod will be considered available as soon as it is ready) + // +optional + MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,5,opt,name=minReadySeconds"` + + // The maximum time in seconds for a deployment to make progress before it + // is considered to be failed. The deployment controller will continue to + // process failed deployments and a condition with a ProgressDeadlineExceeded + // reason will be surfaced in the deployment status. Note that progress will + // not be estimated during the time a deployment is paused. Defaults to 600s. + ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty" protobuf:"varint,9,opt,name=progressDeadlineSeconds"` +} + type RollingStyleType string const ( @@ -59,6 +94,8 @@ const ( PartitionRollingStyle RollingStyleType = "Partition" // CanaryRollingStyle means rolling in canary way, and will create a canary Deployment. CanaryRollingStyle RollingStyleType = "Canary" + // BlueGreenRollingStyle means rolling in blue-green way, and will NOT create a extra Deployment. + BlueGreenRollingStyle RollingStyleType = "BlueGreen" ) // DeploymentExtraStatus is extra status field for Advanced Deployment @@ -74,7 +111,7 @@ type DeploymentExtraStatus struct { } func SetDefaultDeploymentStrategy(strategy *DeploymentStrategy) { - if strategy.RollingStyle == CanaryRollingStyle { + if strategy.RollingStyle != PartitionRollingStyle { return } if strategy.RollingUpdate == nil { @@ -101,3 +138,44 @@ func SetDefaultDeploymentStrategy(strategy *DeploymentStrategy) { } } } + +func SetDefaultSetting(setting *OriginalDeploymentStrategy) { + if setting.ProgressDeadlineSeconds == nil { + setting.ProgressDeadlineSeconds = new(int32) + *setting.ProgressDeadlineSeconds = 600 + } + if setting.Strategy == nil { + setting.Strategy = &apps.DeploymentStrategy{} + } + if setting.Strategy.Type == "" { + setting.Strategy.Type = apps.RollingUpdateDeploymentStrategyType + } + if setting.Strategy.Type == apps.RecreateDeploymentStrategyType { + return + } + strategy := setting.Strategy + if strategy.RollingUpdate == nil { + strategy.RollingUpdate = &apps.RollingUpdateDeployment{} + } + if strategy.RollingUpdate.MaxUnavailable == nil { + // Set MaxUnavailable as 25% by default + maxUnavailable := intstr.FromString("25%") + strategy.RollingUpdate.MaxUnavailable = &maxUnavailable + } + if strategy.RollingUpdate.MaxSurge == nil { + // Set MaxSurge as 25% by default + maxSurge := intstr.FromString("25%") + strategy.RollingUpdate.MaxUnavailable = &maxSurge + } + + // Cannot allow maxSurge==0 && MaxUnavailable==0, otherwise, no pod can be updated when rolling update. + maxSurge, _ := intstr.GetScaledValueFromIntOrPercent(strategy.RollingUpdate.MaxSurge, 100, true) + maxUnavailable, _ := intstr.GetScaledValueFromIntOrPercent(strategy.RollingUpdate.MaxUnavailable, 100, true) + if maxSurge == 0 && maxUnavailable == 0 { + strategy.RollingUpdate = &apps.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + } + } + +} diff --git a/api/v1beta1/rollout_types.go b/api/v1beta1/rollout_types.go index 25c91106..a37b2d83 100644 --- a/api/v1beta1/rollout_types.go +++ b/api/v1beta1/rollout_types.go @@ -75,6 +75,96 @@ type RolloutStrategy struct { Paused bool `json:"paused,omitempty"` // +optional Canary *CanaryStrategy `json:"canary,omitempty"` + // +optional + BlueGreen *BlueGreenStrategy `json:"blueGreen,omitempty" protobuf:"bytes,1,opt,name=blueGreen"` +} + +// Get the rolling style based on the strategy +func (r *RolloutStrategy) GetRollingStyle() RollingStyleType { + if r.BlueGreen != nil { + return BlueGreenRollingStyle + } + //NOTE - even EnableExtraWorkloadForCanary is true, as long as it is not Deployment, + //we won't do canary release. BatchRelease will treat it as Partiton release + if r.Canary.EnableExtraWorkloadForCanary { + return CanaryRollingStyle + } + return PartitionRollingStyle +} + +// r.GetRollingStyle() == BlueGreenRollingStyle +func (r *RolloutStrategy) IsBlueGreenRelease() bool { + return r.GetRollingStyle() == BlueGreenRollingStyle +} + +// r.GetRollingStyle() == CanaryRollingStyle || r.GetRollingStyle() == PartitionRollingStyle +func (r *RolloutStrategy) IsCanaryStragegy() bool { + return r.GetRollingStyle() == CanaryRollingStyle || r.GetRollingStyle() == PartitionRollingStyle +} + +func (r *RolloutStrategy) IsEmptyRelease() bool { + return r.BlueGreen == nil && r.Canary == nil +} + +// Get the steps based on the rolling style +func (r *RolloutStrategy) GetSteps() []CanaryStep { + switch r.GetRollingStyle() { + case BlueGreenRollingStyle: + return r.BlueGreen.Steps + case CanaryRollingStyle, PartitionRollingStyle: + return r.Canary.Steps + default: + return nil + } +} + +// Get the traffic routing based on the rolling style +func (r *RolloutStrategy) GetTrafficRouting() []TrafficRoutingRef { + switch r.GetRollingStyle() { + case BlueGreenRollingStyle: + return r.BlueGreen.TrafficRoutings + case CanaryRollingStyle, PartitionRollingStyle: + return r.Canary.TrafficRoutings + default: + return nil + } +} + +// Check if there are traffic routings +func (r *RolloutStrategy) HasTrafficRoutings() bool { + return len(r.GetTrafficRouting()) > 0 +} + +// Check the value of DisableGenerateCanaryService +func (r *RolloutStrategy) DisableGenerateCanaryService() bool { + switch r.GetRollingStyle() { + case BlueGreenRollingStyle: + return r.BlueGreen.DisableGenerateCanaryService + case CanaryRollingStyle, PartitionRollingStyle: + return r.Canary.DisableGenerateCanaryService + default: + return false + } +} + +// BlueGreenStrategy defines parameters for Blue Green Release +type BlueGreenStrategy struct { + // Steps define the order of phases to execute release in batches(20%, 40%, 60%, 80%, 100%) + // +optional + Steps []CanaryStep `json:"steps,omitempty"` + // TrafficRoutings support ingress, gateway api and custom network resource(e.g. istio, apisix) to enable more fine-grained traffic routing + // and current only support one TrafficRouting + TrafficRoutings []TrafficRoutingRef `json:"trafficRoutings,omitempty"` + // FailureThreshold indicates how many failed pods can be tolerated in all upgraded pods. + // Only when FailureThreshold are satisfied, Rollout can enter ready state. + // If FailureThreshold is nil, Rollout will use the MaxUnavailable of workload as its + // FailureThreshold. + // Defaults to nil. + FailureThreshold *intstr.IntOrString `json:"failureThreshold,omitempty"` + // TrafficRoutingRef is TrafficRouting's Name + TrafficRoutingRef string `json:"trafficRoutingRef,omitempty"` + // canary service will not be generated if DisableGenerateCanaryService is true + DisableGenerateCanaryService bool `json:"disableGenerateCanaryService,omitempty"` } // CanaryStrategy defines parameters for a Replica Based Canary @@ -178,6 +268,9 @@ type RolloutStatus struct { // Canary describes the state of the canary rollout // +optional CanaryStatus *CanaryStatus `json:"canaryStatus,omitempty"` + // BlueGreen describes the state of the blueGreen rollout + // +optional + BlueGreenStatus *BlueGreenStatus `json:"blueGreenStatus,omitempty"` // Conditions a list of conditions a rollout can have. // +optional Conditions []RolloutCondition `json:"conditions,omitempty"` @@ -231,10 +324,26 @@ const ( // Terminating Reason TerminatingReasonInTerminating = "InTerminating" TerminatingReasonCompleted = "Completed" + + // Finalise Reason + // Finalise when the last batch is released and all pods will update to new version + FinaliseReasonSuccess = "Success" + // Finalise when rollback detected + FinaliseReasonRollback = "Rollback" + // Finalise when Continuous Release detected + FinaliseReasonContinuous = "Continuous" + // Finalise when Rollout is disabling + FinaliseReasonDisalbed = "RolloutDisabled" + // Finalise when Rollout is deleting + FinaliseReasonDelete = "RolloutDeleting" ) -// CanaryStatus status fields that only pertain to the canary rollout -type CanaryStatus struct { +// fields in CommonStatus are shared between canary status and bluegreen status +// if a field is accessed in strategy-agnostic way, e.g. accessed from rollout_progressing.go, or rollout_status.go +// then it can be put into CommonStatus +// if a field is only accessed in strategy-specific way, e.g. accessed from rollout_canary.go or rollout_bluegreen.go +// then it should stay behind with CanaryStatus or BlueGreenStatus +type CommonStatus struct { // observedWorkloadGeneration is the most recent generation observed for this Rollout ref workload generation. ObservedWorkloadGeneration int64 `json:"observedWorkloadGeneration,omitempty"` // ObservedRolloutID will record the newest spec.RolloutID if status.canaryRevision equals to workload.updateRevision @@ -243,27 +352,123 @@ type CanaryStatus struct { RolloutHash string `json:"rolloutHash,omitempty"` // StableRevision indicates the revision of stable pods StableRevision string `json:"stableRevision,omitempty"` + // pod template hash is used as service selector label + PodTemplateHash string `json:"podTemplateHash"` + // CurrentStepIndex defines the current step of the rollout is on. If the current step index is null, the + // controller will execute the rollout. + // +optional + CurrentStepIndex int32 `json:"currentStepIndex"` + // NextStepIndex defines the next step of the rollout is on. + // In normal case, NextStepIndex is equal to CurrentStepIndex + 1 + // If the current step is the last step, NextStepIndex is equal to 0 + // Before the release, NextStepIndex is also equal to 0 + // It is allowed to modify NextStepIndex by design, + // e.g. if CurrentStepIndex is 2, user can patch NextStepIndex to 3 (if exists) to + // achieve batch jump, or patch NextStepIndex to 1 to implement a re-execution of step 1 + NextStepIndex int32 `json:"nextStepIndex"` + // FinalisingStep the step of finalising + FinalisingStep FinalisingStepType `json:"finalisingStep"` + CurrentStepState CanaryStepState `json:"currentStepState"` + Message string `json:"message,omitempty"` + LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` +} + +// CanaryStatus status fields that only pertain to the canary rollout +type CanaryStatus struct { + // must be inline + CommonStatus `json:",inline"` // CanaryRevision is calculated by rollout based on podTemplateHash, and the internal logic flow uses // It may be different from rs podTemplateHash in different k8s versions, so it cannot be used as service selector label CanaryRevision string `json:"canaryRevision"` - // pod template hash is used as service selector label - PodTemplateHash string `json:"podTemplateHash"` // CanaryReplicas the numbers of canary revision pods CanaryReplicas int32 `json:"canaryReplicas"` // CanaryReadyReplicas the numbers of ready canary revision pods CanaryReadyReplicas int32 `json:"canaryReadyReplicas"` - // CurrentStepIndex defines the current step of the rollout is on. If the current step index is null, the - // controller will execute the rollout. - // +optional - CurrentStepIndex int32 `json:"currentStepIndex"` - CurrentStepState CanaryStepState `json:"currentStepState"` - Message string `json:"message,omitempty"` - LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` +} + +// BlueGreenStatus status fields that only pertain to the blueGreen rollout +type BlueGreenStatus struct { + CommonStatus `json:",inline"` + // CanaryRevision is calculated by rollout based on podTemplateHash, and the internal logic flow uses + // It may be different from rs podTemplateHash in different k8s versions, so it cannot be used as service selector label + UpdatedRevision string `json:"updatedRevision"` + // UpdatedReplicas the numbers of updated pods + UpdatedReplicas int32 `json:"updatedReplicas"` + // UpdatedReadyReplicas the numbers of updated ready pods + UpdatedReadyReplicas int32 `json:"updatedReadyReplicas"` +} + +// GetSubStatus returns the ethier canary or bluegreen status +func (r *RolloutStatus) GetSubStatus() *CommonStatus { + if r.CanaryStatus != nil { + return &(r.CanaryStatus.CommonStatus) + } + return &(r.BlueGreenStatus.CommonStatus) +} + +func (r *RolloutStatus) IsSubStatusEmpty() bool { + return r.CanaryStatus == nil && r.BlueGreenStatus == nil +} + +func (r *RolloutStatus) Clear() { + r.CanaryStatus = nil + r.BlueGreenStatus = nil +} + +//TODO - the following functions seem awkward, is there better way for our case? + +func (r *RolloutStatus) GetCanaryRevision() string { + if r.CanaryStatus != nil { + return r.CanaryStatus.CanaryRevision + } + return r.BlueGreenStatus.UpdatedRevision +} + +func (r *RolloutStatus) SetCanaryRevision(revision string) { + if r.CanaryStatus != nil { + r.CanaryStatus.CanaryRevision = revision + } + if r.BlueGreenStatus != nil { + r.BlueGreenStatus.UpdatedRevision = revision + } +} + +func (r *RolloutStatus) GetCanaryReplicas() int32 { + if r.CanaryStatus != nil { + return r.CanaryStatus.CanaryReplicas + } + return r.BlueGreenStatus.UpdatedReplicas +} + +func (r *RolloutStatus) SetCanaryReplicas(replicas int32) { + if r.CanaryStatus != nil { + r.CanaryStatus.CanaryReplicas = replicas + } + if r.BlueGreenStatus != nil { + r.BlueGreenStatus.UpdatedReplicas = replicas + } +} + +func (r *RolloutStatus) GetCanaryReadyReplicas() int32 { + if r.CanaryStatus != nil { + return r.CanaryStatus.CanaryReadyReplicas + } + return r.BlueGreenStatus.UpdatedReadyReplicas +} + +func (r *RolloutStatus) SetCanaryReadyReplicas(replicas int32) { + if r.CanaryStatus != nil { + r.CanaryStatus.CanaryReadyReplicas = replicas + } + if r.BlueGreenStatus != nil { + r.BlueGreenStatus.UpdatedReadyReplicas = replicas + } } type CanaryStepState string const ( + CanaryStepStateInit CanaryStepState = "BeforeStepUpgrade" CanaryStepStateUpgrade CanaryStepState = "StepUpgrade" CanaryStepStateTrafficRouting CanaryStepState = "StepTrafficRouting" CanaryStepStateMetricsAnalysis CanaryStepState = "StepMetricsAnalysis" @@ -290,6 +495,34 @@ const ( RolloutPhaseDisabling RolloutPhase = "Disabling" ) +type FinalisingStepType string + +const ( + // some work that should be done before pod scaling down. + // For BlueGreenStrategy: + // we rout all traffic to stable or new version based on FinaliseReason + // For CanaryStrategy: + // we remove the selector of stable service + FinalisingStepTypePreparing FinalisingStepType = "Preparing" + // Patch Batch Release to scale down (exception: the canary Deployment will be + // scaled down in FinalisingStepTypeDeleteBR step) + // For Both BlueGreenStrategy and CanaryStrategy: + // set workload.pause=false, set workload.partition=0 + FinalisingStepTypeBatchRelease FinalisingStepType = "PatchBatchRelease" + //TODO - Currently, the next three steps are in the same function, FinalisingTrafficRouting + // we should try to separate the FinalisingStepTypeGateway and FinalisingStepTypeCanaryService + // with graceful time to prevent some potential issues + + // Restore the stable Service (i.e. remove corresponding selector) + FinalisingStepTypeStableService FinalisingStepType = "RestoreStableService" + // Restore the GatewayAPI/Ingress/Istio + FinalisingStepTypeGateway FinalisingStepType = "RestoreGateway" + // Delete Canary Service + FinalisingStepTypeCanaryService FinalisingStepType = "DeleteCanayService" + // Delete Batch Release + FinalisingStepTypeDeleteBR FinalisingStepType = "DeleteBatchRelease" +) + // +genclient //+kubebuilder:object:root=true //+kubebuilder:subresource:status diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 408b544f..98e46fef 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -157,12 +157,59 @@ func (in *BatchReleaseStatus) DeepCopy() *BatchReleaseStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) { +func (in *BlueGreenStatus) DeepCopyInto(out *BlueGreenStatus) { *out = *in - if in.LastUpdateTime != nil { - in, out := &in.LastUpdateTime, &out.LastUpdateTime - *out = (*in).DeepCopy() + in.CommonStatus.DeepCopyInto(&out.CommonStatus) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlueGreenStatus. +func (in *BlueGreenStatus) DeepCopy() *BlueGreenStatus { + if in == nil { + return nil + } + out := new(BlueGreenStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlueGreenStrategy) DeepCopyInto(out *BlueGreenStrategy) { + *out = *in + if in.Steps != nil { + in, out := &in.Steps, &out.Steps + *out = make([]CanaryStep, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TrafficRoutings != nil { + in, out := &in.TrafficRoutings, &out.TrafficRoutings + *out = make([]TrafficRoutingRef, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.FailureThreshold != nil { + in, out := &in.FailureThreshold, &out.FailureThreshold + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlueGreenStrategy. +func (in *BlueGreenStrategy) DeepCopy() *BlueGreenStrategy { + if in == nil { + return nil } + out := new(BlueGreenStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) { + *out = *in + in.CommonStatus.DeepCopyInto(&out.CommonStatus) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStatus. @@ -236,6 +283,25 @@ func (in *CanaryStrategy) DeepCopy() *CanaryStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommonStatus) DeepCopyInto(out *CommonStatus) { + *out = *in + if in.LastUpdateTime != nil { + in, out := &in.LastUpdateTime, &out.LastUpdateTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonStatus. +func (in *CommonStatus) DeepCopy() *CommonStatus { + if in == nil { + return nil + } + out := new(CommonStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentExtraStatus) DeepCopyInto(out *DeploymentExtraStatus) { *out = *in @@ -344,6 +410,31 @@ func (in *ObjectRef) DeepCopy() *ObjectRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OriginalDeploymentStrategy) DeepCopyInto(out *OriginalDeploymentStrategy) { + *out = *in + if in.Strategy != nil { + in, out := &in.Strategy, &out.Strategy + *out = new(v1.DeploymentStrategy) + (*in).DeepCopyInto(*out) + } + if in.ProgressDeadlineSeconds != nil { + in, out := &in.ProgressDeadlineSeconds, &out.ProgressDeadlineSeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OriginalDeploymentStrategy. +func (in *OriginalDeploymentStrategy) DeepCopy() *OriginalDeploymentStrategy { + if in == nil { + return nil + } + out := new(OriginalDeploymentStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PatchPodTemplateMetadata) DeepCopyInto(out *PatchPodTemplateMetadata) { *out = *in @@ -545,6 +636,11 @@ func (in *RolloutStatus) DeepCopyInto(out *RolloutStatus) { *out = new(CanaryStatus) (*in).DeepCopyInto(*out) } + if in.BlueGreenStatus != nil { + in, out := &in.BlueGreenStatus, &out.BlueGreenStatus + *out = new(BlueGreenStatus) + (*in).DeepCopyInto(*out) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]RolloutCondition, len(*in)) @@ -572,6 +668,11 @@ func (in *RolloutStrategy) DeepCopyInto(out *RolloutStrategy) { *out = new(CanaryStrategy) (*in).DeepCopyInto(*out) } + if in.BlueGreen != nil { + in, out := &in.BlueGreen, &out.BlueGreen + *out = new(BlueGreenStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutStrategy. diff --git a/config/crd/bases/rollouts.kruise.io_batchreleases.yaml b/config/crd/bases/rollouts.kruise.io_batchreleases.yaml index 7eb29e17..ec704f93 100644 --- a/config/crd/bases/rollouts.kruise.io_batchreleases.yaml +++ b/config/crd/bases/rollouts.kruise.io_batchreleases.yaml @@ -118,6 +118,9 @@ spec: description: labels type: object type: object + rollingStyle: + description: RollingStyle can be "Canary", "Partiton" or "BlueGreen" + type: string rolloutID: description: RolloutID indicates an id for each rollout progress type: string @@ -345,13 +348,6 @@ spec: - canaryReplicas type: object type: array - enableExtraWorkloadForCanary: - description: 'If true, then it will create new deployment for - canary, such as: workload-demo-canary. When user verifies that - the canary version is ready, we will remove the canary deployment - and release the deployment workload-demo in full. Current only - support k8s native deployment' - type: boolean failureThreshold: anyOf: - type: integer @@ -382,11 +378,12 @@ spec: description: labels type: object type: object + rollingStyle: + description: RollingStyle can be "Canary", "Partiton" or "BlueGreen" + type: string rolloutID: description: RolloutID indicates an id for each rollout progress type: string - required: - - enableExtraWorkloadForCanary type: object workloadRef: description: WorkloadRef contains enough information to let you identify diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 62c072a6..f257c70f 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -435,18 +435,28 @@ spec: so it cannot be used as service selector label type: string currentStepIndex: - description: CurrentStepIndex defines the current step of the - rollout is on. If the current step index is null, the controller - will execute the rollout. format: int32 type: integer currentStepState: type: string + finalisingStep: + type: string lastUpdateTime: format: date-time type: string message: type: string + nextStepIndex: + description: NextStepIndex defines the next step of the rollout + is on. In normal case, NextStepIndex is equal to CurrentStepIndex + + 1 If the current step is the last step, NextStepIndex is equal + to 0 Before the release, NextStepIndex is also equal to 0 It + is allowed to modify NextStepIndex by design, e.g. if CurrentStepIndex + is 2, user can patch NextStepIndex to 3 (if exists) to achieve + batch jump, or patch NextStepIndex to 1 to implement a re-execution + of step 1 + format: int32 + type: integer observedRolloutID: description: ObservedRolloutID will record the newest spec.RolloutID if status.canaryRevision equals to workload.updateRevision @@ -470,6 +480,8 @@ spec: - canaryReplicas - canaryRevision - currentStepState + - finalisingStep + - nextStepIndex - podTemplateHash type: object conditions: @@ -574,6 +586,308 @@ spec: strategy: description: rollout strategy properties: + blueGreen: + description: BlueGreenStrategy defines parameters for Blue Green + Release + properties: + disableGenerateCanaryService: + description: canary service will not be generated if DisableGenerateCanaryService + is true + type: boolean + failureThreshold: + anyOf: + - type: integer + - type: string + description: FailureThreshold indicates how many failed pods + can be tolerated in all upgraded pods. Only when FailureThreshold + are satisfied, Rollout can enter ready state. If FailureThreshold + is nil, Rollout will use the MaxUnavailable of workload + as its FailureThreshold. Defaults to nil. + x-kubernetes-int-or-string: true + steps: + description: Steps define the order of phases to execute release + in batches(20%, 40%, 60%, 80%, 100%) + items: + description: CanaryStep defines a step of a canary workload. + properties: + matches: + description: Matches define conditions used for matching + the incoming HTTP requests to canary service. Each + match is independent, i.e. this rule will be matched + if **any** one of the matches is satisfied. If Gateway + API, current only support one match. And cannot support + both weight and matches, if both are configured, then + matches takes precedence. + items: + properties: + headers: + description: Headers specifies HTTP request header + matchers. Multiple match values are ANDed together, + meaning, a request must match all the specified + headers to select the route. + items: + description: HTTPHeaderMatch describes how to + select a HTTP route by matching HTTP request + headers. + properties: + name: + description: "Name is the name of the HTTP + Header to be matched. Name matching MUST + be case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify equivalent + header names, only the first entry with + an equivalent name MUST be considered + for a match. Subsequent entries with an + equivalent header name MUST be ignored. + Due to the case-insensitivity of header + names, \"foo\" and \"Foo\" are considered + equivalent. \n When a header is repeated + in an HTTP request, it is implementation-specific + behavior as to how this is represented. + Generally, proxies should follow the guidance + from the RFC: https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 + regarding processing a repeated header, + with special handling for \"Set-Cookie\"." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: "Type specifies how to match + against the value of the header. \n Support: + Core (Exact) \n Support: Custom (RegularExpression) + \n Since RegularExpression HeaderMatchType + has custom conformance, implementations + can support POSIX, PCRE or any other dialects + of regular expressions. Please read the + implementation's documentation to determine + the supported dialect." + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + type: object + type: array + pause: + description: Pause defines a pause stage for a rollout, + manual or auto + properties: + duration: + description: Duration the amount of time to wait + before moving to the next step. + format: int32 + type: integer + type: object + replicas: + anyOf: + - type: integer + - type: string + description: 'Replicas is the number of expected canary + pods in this batch it can be an absolute number (ex: + 5) or a percentage of total pods.' + x-kubernetes-int-or-string: true + requestHeaderModifier: + description: "Set overwrites the request with the given + header (name, value) before the action. \n Input: + \ GET /foo HTTP/1.1 my-header: foo \n requestHeaderModifier: + \ set: - name: \"my-header\" value: \"bar\" + \n Output: GET /foo HTTP/1.1 my-header: bar" + properties: + add: + description: "Add adds the given header(s) (name, + value) to the request before the action. It appends + to any existing values associated with the header + name. \n Input: GET /foo HTTP/1.1 my-header: + foo \n Config: add: - name: \"my-header\" + \ value: \"bar\" \n Output: GET /foo HTTP/1.1 + \ my-header: foo my-header: bar" + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: "Name is the name of the HTTP + Header to be matched. Name matching MUST + be case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify equivalent + header names, the first entry with an equivalent + name MUST be considered for a match. Subsequent + entries with an equivalent header name MUST + be ignored. Due to the case-insensitivity + of header names, \"foo\" and \"Foo\" are + considered equivalent." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: "Remove the given header(s) from the + HTTP request before the action. The value of Remove + is a list of HTTP header names. Note that the + header names are case-insensitive (see https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + \n Input: GET /foo HTTP/1.1 my-header1: foo + \ my-header2: bar my-header3: baz \n Config: + \ remove: [\"my-header1\", \"my-header3\"] \n + Output: GET /foo HTTP/1.1 my-header2: bar" + items: + type: string + maxItems: 16 + type: array + set: + description: "Set overwrites the request with the + given header (name, value) before the action. + \n Input: GET /foo HTTP/1.1 my-header: foo + \n Config: set: - name: \"my-header\" value: + \"bar\" \n Output: GET /foo HTTP/1.1 my-header: + bar" + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: "Name is the name of the HTTP + Header to be matched. Name matching MUST + be case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify equivalent + header names, the first entry with an equivalent + name MUST be considered for a match. Subsequent + entries with an equivalent header name MUST + be ignored. Due to the case-insensitivity + of header names, \"foo\" and \"Foo\" are + considered equivalent." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + traffic: + description: Traffic indicate how many percentage of + traffic the canary pods should receive Value is of + string type and is a percentage, e.g. 5%. + type: string + type: object + type: array + trafficRoutingRef: + description: TrafficRoutingRef is TrafficRouting's Name + type: string + trafficRoutings: + description: TrafficRoutings support ingress, gateway api + and custom network resource(e.g. istio, apisix) to enable + more fine-grained traffic routing and current only support + one TrafficRouting + items: + description: TrafficRoutingRef hosts all the different configuration + for supported service meshes to enable more fine-grained + traffic routing + properties: + customNetworkRefs: + description: CustomNetworkRefs hold a list of custom + providers to route traffic + items: + description: ObjectRef holds a references to the Kubernetes + object + properties: + apiVersion: + description: API Version of the referent + type: string + kind: + description: Kind of the referent + type: string + name: + description: Name of the referent + type: string + required: + - apiVersion + - kind + - name + type: object + type: array + gateway: + description: Gateway holds Gateway specific configuration + to route traffic Gateway configuration only supports + >= v0.4.0 (v1alpha2). + properties: + httpRouteName: + description: HTTPRouteName refers to the name of + an `HTTPRoute` resource in the same namespace + as the `Rollout` + type: string + type: object + gracePeriodSeconds: + description: Optional duration in seconds the traffic + provider(e.g. nginx ingress controller) consumes the + service, ingress configuration changes gracefully. + format: int32 + type: integer + ingress: + description: Ingress holds Ingress specific configuration + to route traffic, e.g. Nginx, Alb. + properties: + classType: + description: ClassType refers to the type of `Ingress`. + current support nginx, aliyun-alb. default is + nginx. + type: string + name: + description: Name refers to the name of an `Ingress` + resource in the same namespace as the `Rollout` + type: string + required: + - name + type: object + service: + description: Service holds the name of a service which + selects pods with stable version and don't select + any pods with canary version. + type: string + required: + - service + type: object + type: array + type: object canary: description: CanaryStrategy defines parameters for a Replica Based Canary @@ -931,6 +1245,78 @@ spec: status: description: RolloutStatus defines the observed state of Rollout properties: + blueGreenStatus: + description: BlueGreen describes the state of the blueGreen rollout + properties: + currentStepIndex: + description: CurrentStepIndex defines the current step of the + rollout is on. If the current step index is null, the controller + will execute the rollout. + format: int32 + type: integer + currentStepState: + type: string + finalisingStep: + description: FinalisingStep the step of finalising + type: string + lastUpdateTime: + format: date-time + type: string + message: + type: string + nextStepIndex: + description: NextStepIndex defines the next step of the rollout + is on. In normal case, NextStepIndex is equal to CurrentStepIndex + + 1 If the current step is the last step, NextStepIndex is equal + to 0 Before the release, NextStepIndex is also equal to 0 It + is allowed to modify NextStepIndex by design, e.g. if CurrentStepIndex + is 2, user can patch NextStepIndex to 3 (if exists) to achieve + batch jump, or patch NextStepIndex to 1 to implement a re-execution + of step 1 + format: int32 + type: integer + observedRolloutID: + description: ObservedRolloutID will record the newest spec.RolloutID + if status.canaryRevision equals to workload.updateRevision + type: string + observedWorkloadGeneration: + description: observedWorkloadGeneration is the most recent generation + observed for this Rollout ref workload generation. + format: int64 + type: integer + podTemplateHash: + description: pod template hash is used as service selector label + type: string + rolloutHash: + description: RolloutHash from rollout.spec object + type: string + stableRevision: + description: StableRevision indicates the revision of stable pods + type: string + updatedReadyReplicas: + description: UpdatedReadyReplicas the numbers of updated ready + pods + format: int32 + type: integer + updatedReplicas: + description: UpdatedReplicas the numbers of updated pods + format: int32 + type: integer + updatedRevision: + description: CanaryRevision is calculated by rollout based on + podTemplateHash, and the internal logic flow uses It may be + different from rs podTemplateHash in different k8s versions, + so it cannot be used as service selector label + type: string + required: + - currentStepState + - finalisingStep + - nextStepIndex + - podTemplateHash + - updatedReadyReplicas + - updatedReplicas + - updatedRevision + type: object canaryStatus: description: Canary describes the state of the canary rollout properties: @@ -957,11 +1343,25 @@ spec: type: integer currentStepState: type: string + finalisingStep: + description: FinalisingStep the step of finalising + type: string lastUpdateTime: format: date-time type: string message: type: string + nextStepIndex: + description: NextStepIndex defines the next step of the rollout + is on. In normal case, NextStepIndex is equal to CurrentStepIndex + + 1 If the current step is the last step, NextStepIndex is equal + to 0 Before the release, NextStepIndex is also equal to 0 It + is allowed to modify NextStepIndex by design, e.g. if CurrentStepIndex + is 2, user can patch NextStepIndex to 3 (if exists) to achieve + batch jump, or patch NextStepIndex to 1 to implement a re-execution + of step 1 + format: int32 + type: integer observedRolloutID: description: ObservedRolloutID will record the newest spec.RolloutID if status.canaryRevision equals to workload.updateRevision @@ -985,6 +1385,8 @@ spec: - canaryReplicas - canaryRevision - currentStepState + - finalisingStep + - nextStepIndex - podTemplateHash type: object conditions: diff --git a/lua_configuration/convert_test_case_to_lua_object.go b/lua_configuration/convert_test_case_to_lua_object.go index 686bea18..d97e959a 100644 --- a/lua_configuration/convert_test_case_to_lua_object.go +++ b/lua_configuration/convert_test_case_to_lua_object.go @@ -81,7 +81,7 @@ func objectToTable(path string) error { rollout := testCase.Rollout trafficRouting := testCase.TrafficRouting if rollout != nil { - steps := rollout.Spec.Strategy.Canary.Steps + steps := rollout.Spec.Strategy.GetSteps() for i, step := range steps { var weight *int32 if step.TrafficRoutingStrategy.Traffic != nil { @@ -92,7 +92,7 @@ func objectToTable(path string) error { weight = utilpointer.Int32(-1) } var canaryService string - stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + stableService := rollout.Spec.Strategy.GetTrafficRouting()[0].Service canaryService = fmt.Sprintf("%s-canary", stableService) data := &custom.LuaData{ Data: custom.Data{ diff --git a/pkg/controller/batchrelease/batchrelease_controller_test.go b/pkg/controller/batchrelease/batchrelease_controller_test.go index c339b74b..8f79493e 100644 --- a/pkg/controller/batchrelease/batchrelease_controller_test.go +++ b/pkg/controller/batchrelease/batchrelease_controller_test.go @@ -67,8 +67,8 @@ var ( Name: "sample", }, ReleasePlan: v1beta1.ReleasePlan{ - EnableExtraWorkloadForCanary: true, - BatchPartition: pointer.Int32(0), + RollingStyle: v1beta1.CanaryRollingStyle, + BatchPartition: pointer.Int32(0), Batches: []v1beta1.ReleaseBatch{ { CanaryReplicas: intstr.FromString("10%"), @@ -147,6 +147,7 @@ var ( }, ReleasePlan: v1beta1.ReleasePlan{ BatchPartition: pointer.Int32Ptr(0), + RollingStyle: v1beta1.PartitionRollingStyle, Batches: []v1beta1.ReleaseBatch{ { CanaryReplicas: intstr.FromString("10%"), diff --git a/pkg/controller/batchrelease/batchrelease_executor.go b/pkg/controller/batchrelease/batchrelease_executor.go index e1aa563e..267239ea 100644 --- a/pkg/controller/batchrelease/batchrelease_executor.go +++ b/pkg/controller/batchrelease/batchrelease_executor.go @@ -198,27 +198,40 @@ func (r *Executor) getReleaseController(release *v1beta1.BatchRelease, newStatus Name: targetRef.Name, } - switch targetRef.APIVersion { - case appsv1alpha1.GroupVersion.String(): - if targetRef.Kind == reflect.TypeOf(appsv1alpha1.CloneSet{}).Name() { + rollingStyle := release.Spec.ReleasePlan.RollingStyle + klog.Infof("BatchRelease(%v) using %s-style release controller for this batch release", klog.KObj(release), rollingStyle) + switch rollingStyle { + case v1beta1.BlueGreenRollingStyle: + // if targetRef.APIVersion == appsv1alpha1.GroupVersion.String() && targetRef.Kind == reflect.TypeOf(appsv1alpha1.CloneSet{}).Name() { + // klog.InfoS("Using CloneSet bluegreen-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) + // return partitionstyle.NewControlPlane(cloneset.NewController, r.client, r.recorder, release, newStatus, targetKey, gvk), nil + // } + // if targetRef.APIVersion == apps.SchemeGroupVersion.String() && targetRef.Kind == reflect.TypeOf(apps.Deployment{}).Name() { + // klog.InfoS("Using Deployment bluegreen-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) + // return bluegreenstyle.NewControlPlane(deployment.NewController, r.client, r.recorder, release, newStatus, targetKey, gvk), nil + // } + + case v1beta1.CanaryRollingStyle: + if targetRef.APIVersion == apps.SchemeGroupVersion.String() && targetRef.Kind == reflect.TypeOf(apps.Deployment{}).Name() { + klog.InfoS("Using Deployment canary-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) + return canarystyle.NewControlPlane(canarydeployment.NewController, r.client, r.recorder, release, newStatus, targetKey), nil + } + fallthrough + + case v1beta1.PartitionRollingStyle, "": + if targetRef.APIVersion == appsv1alpha1.GroupVersion.String() && targetRef.Kind == reflect.TypeOf(appsv1alpha1.CloneSet{}).Name() { klog.InfoS("Using CloneSet partition-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) return partitionstyle.NewControlPlane(cloneset.NewController, r.client, r.recorder, release, newStatus, targetKey, gvk), nil } - if targetRef.Kind == reflect.TypeOf(appsv1alpha1.DaemonSet{}).Name() { + if targetRef.APIVersion == appsv1alpha1.GroupVersion.String() && targetRef.Kind == reflect.TypeOf(appsv1alpha1.DaemonSet{}).Name() { klog.InfoS("Using DaemonSet partition-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) return partitionstyle.NewControlPlane(daemonset.NewController, r.client, r.recorder, release, newStatus, targetKey, gvk), nil } - - case apps.SchemeGroupVersion.String(): - if targetRef.Kind == reflect.TypeOf(apps.Deployment{}).Name() { - if !release.Spec.ReleasePlan.EnableExtraWorkloadForCanary { - klog.InfoS("Using Deployment partition-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) - return partitionstyle.NewControlPlane(partitiondeployment.NewController, r.client, r.recorder, release, newStatus, targetKey, gvk), nil - } else { - klog.InfoS("Using Deployment canary-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) - return canarystyle.NewControlPlane(canarydeployment.NewController, r.client, r.recorder, release, newStatus, targetKey), nil - } + if targetRef.APIVersion == apps.SchemeGroupVersion.String() && targetRef.Kind == reflect.TypeOf(apps.Deployment{}).Name() { + klog.InfoS("Using Deployment partition-style release controller for this batch release", "workload name", targetKey.Name, "namespace", targetKey.Namespace) + return partitionstyle.NewControlPlane(partitiondeployment.NewController, r.client, r.recorder, release, newStatus, targetKey, gvk), nil } + klog.Info("Partition, but use StatefulSet-Like partition-style release controller for this batch release") } // try to use StatefulSet-like rollout controller by default diff --git a/pkg/controller/batchrelease/context/context.go b/pkg/controller/batchrelease/context/context.go index 5a801249..2c428180 100644 --- a/pkg/controller/batchrelease/context/context.go +++ b/pkg/controller/batchrelease/context/context.go @@ -73,20 +73,20 @@ func (bc *BatchContext) Log() string { // IsBatchReady return nil if the batch is ready func (bc *BatchContext) IsBatchReady() error { if bc.UpdatedReplicas < bc.DesiredUpdatedReplicas { - return fmt.Errorf("current batch not ready: updated replicas not satified") + return fmt.Errorf("current batch not ready: updated replicas not satisfied, UpdatedReplicas %d < DesiredUpdatedReplicas %d", bc.UpdatedReplicas, bc.DesiredUpdatedReplicas) } unavailableToleration := allowedUnavailable(bc.FailureThreshold, bc.UpdatedReplicas) if unavailableToleration+bc.UpdatedReadyReplicas < bc.DesiredUpdatedReplicas { - return fmt.Errorf("current batch not ready: updated ready replicas not satified") + return fmt.Errorf("current batch not ready: updated ready replicas not satisfied, allowedUnavailable + UpdatedReadyReplicas %d < DesiredUpdatedReplicas %d", unavailableToleration+bc.UpdatedReadyReplicas, bc.DesiredUpdatedReplicas) } if bc.DesiredUpdatedReplicas > 0 && bc.UpdatedReadyReplicas == 0 { - return fmt.Errorf("current batch not ready: no updated ready replicas") + return fmt.Errorf("current batch not ready: no updated ready replicas, DesiredUpdatedReplicas %d > 0 and UpdatedReadyReplicas %d = 0", bc.DesiredUpdatedReplicas, bc.UpdatedReadyReplicas) } if !batchLabelSatisfied(bc.Pods, bc.RolloutID, bc.PlannedUpdatedReplicas) { - return fmt.Errorf("current batch not ready: pods with batch label not satified") + return fmt.Errorf("current batch not ready: pods with batch label not satisfied, RolloutID %s, PlannedUpdatedReplicas %d", bc.RolloutID, bc.PlannedUpdatedReplicas) } return nil } diff --git a/pkg/controller/rollout/rollout_canary.go b/pkg/controller/rollout/rollout_canary.go index 41ed5f90..0c3e62f4 100644 --- a/pkg/controller/rollout/rollout_canary.go +++ b/pkg/controller/rollout/rollout_canary.go @@ -87,6 +87,13 @@ func (m *canaryReleaseManager) runCanary(c *RolloutContext) error { } } switch canaryStatus.CurrentStepState { + // before CanaryStepStateUpgrade, handle some special cases, to prevent traffic loss + case v1beta1.CanaryStepStateInit: + // placeholder for the later traffic modification Pull Request + canaryStatus.NextStepIndex = util.NextBatchIndex(c.Rollout, canaryStatus.CurrentStepIndex) + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + fallthrough + case v1beta1.CanaryStepStateUpgrade: klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1beta1.CanaryStepStateUpgrade) done, err := m.doCanaryUpgrade(c) @@ -144,7 +151,8 @@ func (m *canaryReleaseManager) runCanary(c *RolloutContext) error { if len(c.Rollout.Spec.Strategy.Canary.Steps) > int(canaryStatus.CurrentStepIndex) { canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} canaryStatus.CurrentStepIndex++ - canaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + canaryStatus.NextStepIndex = util.NextBatchIndex(c.Rollout, canaryStatus.CurrentStepIndex) + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) } else { klog.Infof("rollout(%s/%s) canary run all steps, and completed", c.Rollout.Namespace, c.Rollout.Name) @@ -201,15 +209,13 @@ func (m *canaryReleaseManager) doCanaryMetricsAnalysis(c *RolloutContext) (bool, } func (m *canaryReleaseManager) doCanaryPaused(c *RolloutContext) (bool, error) { + if m.doCanaryJump(c) { + klog.Infof("rollout(%s/%s) canary step jumped", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } canaryStatus := c.NewStatus.CanaryStatus currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] steps := len(c.Rollout.Spec.Strategy.Canary.Steps) - // If it is the last step, and 100% of pods, then return true - if int32(steps) == canaryStatus.CurrentStepIndex { - if currentStep.Replicas != nil && currentStep.Replicas.StrVal == "100%" { - return true, nil - } - } cond := util.GetRolloutCondition(*c.NewStatus, v1beta1.RolloutConditionProgressing) // need manual confirmation if currentStep.Pause.Duration == nil { @@ -232,6 +238,34 @@ func (m *canaryReleaseManager) doCanaryPaused(c *RolloutContext) (bool, error) { return false, nil } +func (m *canaryReleaseManager) doCanaryJump(c *RolloutContext) (jumped bool) { + canaryStatus := c.NewStatus.CanaryStatus + currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] + nextIndex := canaryStatus.NextStepIndex + if nextIndex != util.NextBatchIndex(c.Rollout, canaryStatus.CurrentStepIndex) && nextIndex > 0 { + currentIndexBackup := canaryStatus.CurrentStepIndex + canaryStatus.CurrentStepIndex = nextIndex + canaryStatus.NextStepIndex = util.NextBatchIndex(c.Rollout, nextIndex) + nextStep := c.Rollout.Spec.Strategy.Canary.Steps[nextIndex-1] + // if the Replicas between currentStep and nextStep is same, we can jump to + // the TrafficRouting step; otherwise, we should start from the Init step + if reflect.DeepEqual(nextStep.Replicas, currentStep.Replicas) { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateTrafficRouting + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1beta1.CanaryStepStatePaused, canaryStatus.CurrentStepState) + } else { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1beta1.CanaryStepStatePaused, v1beta1.CanaryStepStateInit) + } + klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", c.Rollout.Namespace, c.Rollout.Name, currentIndexBackup, canaryStatus.CurrentStepIndex) + return true + } + return false +} + // cleanup after rollout is completed or finished func (m *canaryReleaseManager) doCanaryFinalising(c *RolloutContext) (bool, error) { // when CanaryStatus is nil, which means canary action hasn't started yet, don't need doing cleanup @@ -359,12 +393,12 @@ func createBatchRelease(rollout *v1beta1.Rollout, rolloutID string, batch int32, Name: rollout.Spec.WorkloadRef.Name, }, ReleasePlan: v1beta1.ReleasePlan{ - Batches: batches, - RolloutID: rolloutID, - BatchPartition: utilpointer.Int32Ptr(batch), - FailureThreshold: rollout.Spec.Strategy.Canary.FailureThreshold, - PatchPodTemplateMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, - EnableExtraWorkloadForCanary: rollout.Spec.Strategy.Canary.EnableExtraWorkloadForCanary, + Batches: batches, + RolloutID: rolloutID, + BatchPartition: utilpointer.Int32Ptr(batch), + FailureThreshold: rollout.Spec.Strategy.Canary.FailureThreshold, + PatchPodTemplateMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, + RollingStyle: rollout.Spec.Strategy.GetRollingStyle(), }, }, } diff --git a/pkg/controller/rollout/rollout_canary_test.go b/pkg/controller/rollout/rollout_canary_test.go index 89148109..573849cf 100644 --- a/pkg/controller/rollout/rollout_canary_test.go +++ b/pkg/controller/rollout/rollout_canary_test.go @@ -99,7 +99,7 @@ func TestRunCanary(t *testing.T) { }, } br.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - br.Spec.ReleasePlan.EnableExtraWorkloadForCanary = true + br.Spec.ReleasePlan.RollingStyle = v1beta1.CanaryRollingStyle return br }, }, @@ -158,10 +158,12 @@ func TestRunCanary(t *testing.T) { }, } br.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - br.Spec.ReleasePlan.EnableExtraWorkloadForCanary = true + br.Spec.ReleasePlan.RollingStyle = v1beta1.CanaryRollingStyle br.Status = v1beta1.BatchReleaseStatus{ - ObservedGeneration: 1, - ObservedReleasePlanHash: "d444a1007776da957d7d8549e3375c96179621b85670ad1e2bb0fc5fea16446a", + ObservedGeneration: 1, + // since we use RollingStyle over EnableExtraWorkloadForCanary now, former hardcoded hash + // should be re-calculated + ObservedReleasePlanHash: util.HashReleasePlanBatches(&br.Spec.ReleasePlan), CanaryStatus: v1beta1.BatchReleaseCanaryStatus{ CurrentBatchState: v1beta1.ReadyBatchState, CurrentBatch: 0, @@ -204,7 +206,7 @@ func TestRunCanary(t *testing.T) { }, } br.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - br.Spec.ReleasePlan.EnableExtraWorkloadForCanary = true + br.Spec.ReleasePlan.RollingStyle = v1beta1.CanaryRollingStyle return br }, }, diff --git a/pkg/controller/rollout/rollout_progressing.go b/pkg/controller/rollout/rollout_progressing.go index 0aaf5975..1928d432 100644 --- a/pkg/controller/rollout/rollout_progressing.go +++ b/pkg/controller/rollout/rollout_progressing.go @@ -66,17 +66,38 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1beta1.Rollout switch cond.Reason { case v1alpha1.ProgressingReasonInitializing: klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) - // new canaryStatus - newStatus.CanaryStatus = &v1beta1.CanaryStatus{ - ObservedWorkloadGeneration: rolloutContext.Workload.Generation, - RolloutHash: rolloutContext.Rollout.Annotations[util.RolloutHashAnnotation], - ObservedRolloutID: getRolloutID(rolloutContext.Workload), - StableRevision: rolloutContext.Workload.StableRevision, - CanaryRevision: rolloutContext.Workload.CanaryRevision, - CurrentStepIndex: 1, - CurrentStepState: v1beta1.CanaryStepStateUpgrade, - LastUpdateTime: &metav1.Time{Time: time.Now()}, + // clear and create + newStatus.Clear() + if rollout.Spec.Strategy.IsBlueGreenRelease() { + newStatus.BlueGreenStatus = &v1beta1.BlueGreenStatus{ + CommonStatus: v1beta1.CommonStatus{ + ObservedWorkloadGeneration: rolloutContext.Workload.Generation, + RolloutHash: rolloutContext.Rollout.Annotations[util.RolloutHashAnnotation], + ObservedRolloutID: getRolloutID(rolloutContext.Workload), + StableRevision: rolloutContext.Workload.StableRevision, + CurrentStepIndex: 1, + NextStepIndex: util.NextBatchIndex(rollout, 1), + CurrentStepState: v1beta1.CanaryStepStateInit, + LastUpdateTime: &metav1.Time{Time: time.Now()}, + }, + UpdatedRevision: rolloutContext.Workload.CanaryRevision, + } + } else { + newStatus.CanaryStatus = &v1beta1.CanaryStatus{ + CommonStatus: v1beta1.CommonStatus{ + ObservedWorkloadGeneration: rolloutContext.Workload.Generation, + RolloutHash: rolloutContext.Rollout.Annotations[util.RolloutHashAnnotation], + ObservedRolloutID: getRolloutID(rolloutContext.Workload), + StableRevision: rolloutContext.Workload.StableRevision, + CurrentStepIndex: 1, + NextStepIndex: util.NextBatchIndex(rollout, 1), + CurrentStepState: v1beta1.CanaryStepStateInit, + LastUpdateTime: &metav1.Time{Time: time.Now()}, + }, + CanaryRevision: rolloutContext.Workload.CanaryRevision, + } } + done, err := r.doProgressingInitializing(rolloutContext) if err != nil { klog.Errorf("rollout(%s/%s) doProgressingInitializing error(%s)", rollout.Namespace, rollout.Name, err.Error()) @@ -150,7 +171,7 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1beta1.Rollout func (r *RolloutReconciler) doProgressingInitializing(c *RolloutContext) (bool, error) { // Traffic routing - if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if c.Rollout.Spec.Strategy.HasTrafficRoutings() { if err := r.trafficRoutingManager.InitializeTrafficRouting(newTrafficRoutingContext(c)); err != nil { return false, err } @@ -209,14 +230,15 @@ func (r *RolloutReconciler) handleRolloutPaused(rollout *v1beta1.Rollout, newSta func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error { r.Recorder.Eventf(c.Rollout, corev1.EventTypeNormal, "Progressing", "workload continuous publishing canaryRevision, then restart publishing") klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", - c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CanaryRevision, c.Workload.CanaryRevision) + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetCanaryRevision(), c.Workload.CanaryRevision) done, err := r.doProgressingReset(c) if err != nil { klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) return err } else if done { - c.NewStatus.CanaryStatus = nil + // clear SubStatus + c.NewStatus.Clear() progressingStateTransition(c.NewStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonInitializing, "Workload is continuous release") klog.Infof("rollout(%s/%s) workload is continuous publishing, reset complete", c.Rollout.Namespace, c.Rollout.Name) } else { @@ -229,7 +251,7 @@ func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error { } func (r *RolloutReconciler) handleRollbackDirectly(rollout *v1beta1.Rollout, workload *util.Workload, newStatus *v1beta1.RolloutStatus) error { - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision + newStatus.SetCanaryRevision(workload.CanaryRevision) r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") klog.Infof("rollout(%s/%s) workload has been rollback directly, then rollout canceled", rollout.Namespace, rollout.Name) progressingStateTransition(newStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") @@ -238,11 +260,12 @@ func (r *RolloutReconciler) handleRollbackDirectly(rollout *v1beta1.Rollout, wor func (r *RolloutReconciler) handleRollbackInBatches(rollout *v1beta1.Rollout, workload *util.Workload, newStatus *v1beta1.RolloutStatus) error { // restart from the beginning - newStatus.CanaryStatus.CurrentStepIndex = 1 - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision - newStatus.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade - newStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - newStatus.CanaryStatus.RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] + newStatus.GetSubStatus().CurrentStepIndex = 1 + newStatus.GetSubStatus().NextStepIndex = util.NextBatchIndex(rollout, 1) + newStatus.SetCanaryRevision(workload.CanaryRevision) + newStatus.GetSubStatus().CurrentStepState = v1beta1.CanaryStepStateInit + newStatus.GetSubStatus().LastUpdateTime = &metav1.Time{Time: time.Now()} + newStatus.GetSubStatus().RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] klog.Infof("rollout(%s/%s) workload has been rollback in batches, then restart from beginning", rollout.Namespace, rollout.Name) return nil } @@ -253,24 +276,42 @@ func (r *RolloutReconciler) handleRolloutPlanChanged(c *RolloutContext) error { klog.Errorf("rollout(%s/%s) reCalculate Canary StepIndex failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) return err } - // canary step configuration change causes current step index change - c.NewStatus.CanaryStatus.CurrentStepIndex = newStepIndex - c.NewStatus.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade - c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - c.NewStatus.CanaryStatus.RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] - klog.Infof("rollout(%s/%s) canary step configuration change, and stepIndex(%d) state(%s)", - c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CurrentStepIndex, c.NewStatus.CanaryStatus.CurrentStepState) + // if the target step index is the same as the NextStepIndex + // we simply set the CurrentStepState to Ready + if c.NewStatus.GetSubStatus().NextStepIndex == newStepIndex { + c.NewStatus.GetSubStatus().CurrentStepState = v1beta1.CanaryStepStateReady + c.NewStatus.GetSubStatus().LastUpdateTime = &metav1.Time{Time: time.Now()} + c.NewStatus.GetSubStatus().RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] + klog.Infof("rollout(%s/%s) canary step configuration change, and NextStepIndex(%d) state(%s)", + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetSubStatus().NextStepIndex, c.NewStatus.GetSubStatus().CurrentStepState) + return nil + } + + // otherwise, we jump to step paused, where the "jump" logic exists + c.NewStatus.GetSubStatus().NextStepIndex = newStepIndex + c.NewStatus.GetSubStatus().CurrentStepState = v1beta1.CanaryStepStatePaused + c.NewStatus.GetSubStatus().LastUpdateTime = &metav1.Time{Time: time.Now()} + c.NewStatus.GetSubStatus().RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] + klog.Infof("rollout(%s/%s) canary step configuration change, and NextStepIndex(%d) state(%s)", + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetSubStatus().NextStepIndex, c.NewStatus.GetSubStatus().CurrentStepState) return nil } func (r *RolloutReconciler) handleNormalRolling(c *RolloutContext) error { // check if canary is done - if c.NewStatus.CanaryStatus.CurrentStepState == v1beta1.CanaryStepStateCompleted { + if c.NewStatus.GetSubStatus().CurrentStepState == v1beta1.CanaryStepStateCompleted { klog.Infof("rollout(%s/%s) progressing rolling done", c.Rollout.Namespace, c.Rollout.Name) progressingStateTransition(c.NewStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonFinalising, "Rollout has been completed and some closing work is being done") return nil } - return r.canaryManager.runCanary(c) + // in case user modifies it with inappropriate value + util.CheckNextBatchIndexWithCorrect(c.Rollout) + + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return err + } + return releaseManager.runCanary(c) } // name is rollout name, tr is trafficRouting name @@ -330,30 +371,41 @@ func (r *RolloutReconciler) finalizeTrafficRouting(namespace, name, tr string) e *********************************************************************** */ + +func (r *RolloutReconciler) getReleaseManager(rollout *v1beta1.Rollout) (ReleaseManager, error) { + if rollout.Spec.Strategy.IsCanaryStragegy() { + return r.canaryManager, nil + } else if rollout.Spec.Strategy.IsBlueGreenRelease() { + // placeholder for upcoming PR + // return r.blueGreenManager, nil + } + return nil, fmt.Errorf("unknown rolling style: %s, and thus cannot call corresponding release manager", rollout.Spec.Strategy.GetRollingStyle()) +} + func isRolloutPaused(rollout *v1beta1.Rollout) bool { return rollout.Spec.Strategy.Paused } func isRolloutPlanChanged(rollout *v1beta1.Rollout) bool { status := &rollout.Status - return status.CanaryStatus.RolloutHash != "" && status.CanaryStatus.RolloutHash != rollout.Annotations[util.RolloutHashAnnotation] + return status.GetSubStatus().RolloutHash != "" && status.GetSubStatus().RolloutHash != rollout.Annotations[util.RolloutHashAnnotation] } func isContinuousRelease(rollout *v1beta1.Rollout, workload *util.Workload) bool { status := &rollout.Status - return status.CanaryStatus.CanaryRevision != "" && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !workload.IsInRollback + return status.GetCanaryRevision() != "" && workload.CanaryRevision != status.GetCanaryRevision() && !workload.IsInRollback } func isRollingBackDirectly(rollout *v1beta1.Rollout, workload *util.Workload) bool { status := &rollout.Status inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) - return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !inBatch + return workload.IsInRollback && workload.CanaryRevision != status.GetCanaryRevision() && !inBatch } func isRollingBackInBatches(rollout *v1beta1.Rollout, workload *util.Workload) bool { status := &rollout.Status inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) - return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && inBatch + return workload.IsInRollback && workload.CanaryRevision != status.GetCanaryRevision() && inBatch } // 1. modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods @@ -379,16 +431,35 @@ func (r *RolloutReconciler) doProgressingReset(c *RolloutContext) (bool, error) } func (r *RolloutReconciler) recalculateCanaryStep(c *RolloutContext) (int32, error) { - batch, err := r.canaryManager.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return 0, err + } + batch, err := releaseManager.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) if errors.IsNotFound(err) { return 1, nil } else if err != nil { return 0, err } currentReplicas, _ := intstr.GetScaledValueFromIntOrPercent(&batch.Spec.ReleasePlan.Batches[*batch.Spec.ReleasePlan.BatchPartition].CanaryReplicas, int(c.Workload.Replicas), true) - var stepIndex int32 - for i := range c.Rollout.Spec.Strategy.Canary.Steps { - step := c.Rollout.Spec.Strategy.Canary.Steps[i] + var stepIndex, currentIndex int32 + if c.NewStatus != nil { + currentIndex = c.NewStatus.GetSubStatus().CurrentStepIndex - 1 + } + steps := append([]int{}, int(currentIndex)) + // we don't distinguish between the changes in Replicas and Traffic + // Whatever the change is, we recalculate the step. + // we put the current step index first for retrieval, so that if Traffic is the only change, + // usually we will get the target step index same as current step index + for i := 0; i < len(c.Rollout.Spec.Strategy.GetSteps()); i++ { + if i == int(currentIndex) { + continue + } + steps = append(steps, i) + } + + for _, i := range steps { + step := c.Rollout.Spec.Strategy.GetSteps()[i] var desiredReplicas int desiredReplicas, _ = intstr.GetScaledValueFromIntOrPercent(step.Replicas, int(c.Workload.Replicas), true) stepIndex = int32(i + 1) @@ -396,6 +467,7 @@ func (r *RolloutReconciler) recalculateCanaryStep(c *RolloutContext) (int32, err break } } + klog.Infof("RolloutPlan Change detected, rollout(%s/%s) currentStepIndex %d, jumps to %d", c.Rollout.Namespace, c.Rollout.Name, currentIndex+1, stepIndex) return stepIndex, nil } @@ -408,7 +480,11 @@ func (r *RolloutReconciler) doFinalising(c *RolloutContext) (bool, error) { return false, err } } - done, err := r.canaryManager.doCanaryFinalising(c) + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return false, err + } + done, err := releaseManager.doCanaryFinalising(c) if err != nil { klog.Errorf("rollout(%s/%s) Progressing failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) return false, err @@ -446,7 +522,17 @@ func setRolloutSucceededCondition(status *v1beta1.RolloutStatus, condStatus core } func newTrafficRoutingContext(c *RolloutContext) *trafficrouting.TrafficRoutingContext { - currentStep := c.Rollout.Spec.Strategy.Canary.Steps[c.NewStatus.CanaryStatus.CurrentStepIndex-1] + currentIndex := c.NewStatus.GetSubStatus().CurrentStepIndex - 1 + var currentStep v1beta1.CanaryStep + //TODO - need better designed logic + if currentIndex < 0 || int(currentIndex) >= len(c.Rollout.Spec.Strategy.GetSteps()) { + klog.Warningf("Rollout(%s/%s) encounters a special case when constructing newTrafficRoutingContext", c.Rollout.Namespace, c.Rollout.Name) + // usually this only happens when deleting the rollout or rolling back + // in this situation, it's no matter which step the current is + currentStep = c.Rollout.Spec.Strategy.GetSteps()[0] + } else { + currentStep = c.Rollout.Spec.Strategy.GetSteps()[currentIndex] + } var revisionLabelKey string if c.Workload != nil { revisionLabelKey = c.Workload.RevisionLabelKey @@ -454,13 +540,13 @@ func newTrafficRoutingContext(c *RolloutContext) *trafficrouting.TrafficRoutingC return &trafficrouting.TrafficRoutingContext{ Key: fmt.Sprintf("Rollout(%s/%s)", c.Rollout.Namespace, c.Rollout.Name), Namespace: c.Rollout.Namespace, - ObjectRef: c.Rollout.Spec.Strategy.Canary.TrafficRoutings, + ObjectRef: c.Rollout.Spec.Strategy.GetTrafficRouting(), Strategy: currentStep.TrafficRoutingStrategy, OwnerRef: *metav1.NewControllerRef(c.Rollout, rolloutControllerKind), RevisionLabelKey: revisionLabelKey, - StableRevision: c.NewStatus.CanaryStatus.StableRevision, - CanaryRevision: c.NewStatus.CanaryStatus.PodTemplateHash, - LastUpdateTime: c.NewStatus.CanaryStatus.LastUpdateTime, - DisableGenerateCanaryService: c.Rollout.Spec.Strategy.Canary.DisableGenerateCanaryService, + StableRevision: c.NewStatus.GetSubStatus().StableRevision, + CanaryRevision: c.NewStatus.GetSubStatus().PodTemplateHash, + LastUpdateTime: c.NewStatus.GetSubStatus().LastUpdateTime, + DisableGenerateCanaryService: c.Rollout.Spec.Strategy.DisableGenerateCanaryService(), } } diff --git a/pkg/controller/rollout/rollout_progressing_test.go b/pkg/controller/rollout/rollout_progressing_test.go index e9649394..6513faff 100644 --- a/pkg/controller/rollout/rollout_progressing_test.go +++ b/pkg/controller/rollout/rollout_progressing_test.go @@ -67,7 +67,11 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.StableRevision = "pod-template-hash-v1" s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.CurrentStepIndex = 1 - s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + // s.CanaryStatus.NextStepIndex will be initialized as 0 in ReconcileRolloutProgressing. + // util.NextBatchIndex(rollout, s.CanaryStatus.CurrentStepIndex), which is 2 here. + s.CanaryStatus.NextStepIndex = 2 + // now the first step is no longer StepStateUpgrade, it is StepStateInit now + s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit return s }, expectTr: func() *v1alpha1.TrafficRouting { @@ -99,7 +103,8 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.StableRevision = "pod-template-hash-v1" s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.CurrentStepIndex = 1 - s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + s.CanaryStatus.NextStepIndex = 2 + s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(s, *cond) @@ -140,6 +145,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" obj.Status.CanaryStatus.CanaryRevision = "6f8cc56547" obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.NextStepIndex = 2 obj.Status.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade cond := util.GetRolloutCondition(obj.Status, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling @@ -154,6 +160,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.NextStepIndex = 2 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling @@ -210,6 +217,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonFinalising @@ -268,6 +276,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonFinalising @@ -328,6 +337,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond2 := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond2.Reason = v1alpha1.ProgressingReasonFinalising @@ -377,6 +387,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond2 := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond2.Reason = v1alpha1.ProgressingReasonCompleted diff --git a/pkg/controller/rollout/rollout_releaseManager.go b/pkg/controller/rollout/rollout_releaseManager.go new file mode 100644 index 00000000..25cb199b --- /dev/null +++ b/pkg/controller/rollout/rollout_releaseManager.go @@ -0,0 +1,12 @@ +package rollout + +import ( + "github.com/openkruise/rollouts/api/v1beta1" +) + +type ReleaseManager interface { + runCanary(c *RolloutContext) error + doCanaryFinalising(c *RolloutContext) (bool, error) + fetchBatchRelease(ns, name string) (*v1beta1.BatchRelease, error) + removeBatchRelease(c *RolloutContext) (bool, error) +} diff --git a/pkg/controller/rollout/rollout_status.go b/pkg/controller/rollout/rollout_status.go index 8165bcc4..a3629413 100755 --- a/pkg/controller/rollout/rollout_status.go +++ b/pkg/controller/rollout/rollout_status.go @@ -91,10 +91,10 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re // update workload generation to canaryStatus.ObservedWorkloadGeneration // rollout is a target ref bypass, so there needs to be a field to identify the rollout execution process or results, // which version of deployment is targeted, ObservedWorkloadGeneration that is to compare with the workload generation - if newStatus.CanaryStatus != nil && newStatus.CanaryStatus.CanaryRevision != "" && - newStatus.CanaryStatus.CanaryRevision == workload.CanaryRevision { - newStatus.CanaryStatus.ObservedRolloutID = getRolloutID(workload) - newStatus.CanaryStatus.ObservedWorkloadGeneration = workload.Generation + if !newStatus.IsSubStatusEmpty() && newStatus.GetCanaryRevision() != "" && + newStatus.GetCanaryRevision() == workload.CanaryRevision { + newStatus.GetSubStatus().ObservedRolloutID = getRolloutID(workload) + newStatus.GetSubStatus().ObservedWorkloadGeneration = workload.Generation } switch newStatus.Phase { @@ -110,7 +110,7 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re cond := util.NewRolloutCondition(v1beta1.RolloutConditionProgressing, corev1.ConditionTrue, v1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing") util.SetRolloutCondition(newStatus, *cond) util.RemoveRolloutCondition(newStatus, v1beta1.RolloutConditionSucceeded) - } else if newStatus.CanaryStatus == nil { + } else if newStatus.IsSubStatusEmpty() { // The following logic is to make PaaS be able to judge whether the rollout is ready // at the first deployment of the Rollout/Workload. For example: generally, a PaaS // platform can use the following code to judge whether the rollout progression is completed: @@ -123,17 +123,36 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re // But at the first deployment of Rollout/Workload, CanaryStatus isn't set due to no rollout progression, // and PaaS platform cannot judge whether the deployment is completed base on the code above. So we have // to update the status just like the rollout was completed. - - newStatus.CanaryStatus = &v1beta1.CanaryStatus{ - ObservedRolloutID: getRolloutID(workload), - ObservedWorkloadGeneration: workload.Generation, - PodTemplateHash: workload.PodTemplateHash, - CanaryRevision: workload.CanaryRevision, - StableRevision: workload.StableRevision, - CurrentStepIndex: int32(len(rollout.Spec.Strategy.Canary.Steps)), - CurrentStepState: v1beta1.CanaryStepStateCompleted, - RolloutHash: rollout.Annotations[util.RolloutHashAnnotation], + if rollout.Spec.Strategy.IsBlueGreenRelease() { + newStatus.BlueGreenStatus = &v1beta1.BlueGreenStatus{ + CommonStatus: v1beta1.CommonStatus{ + ObservedRolloutID: getRolloutID(workload), + ObservedWorkloadGeneration: workload.Generation, + PodTemplateHash: workload.PodTemplateHash, + StableRevision: workload.StableRevision, + CurrentStepIndex: int32(len(rollout.Spec.Strategy.GetSteps())), + NextStepIndex: util.NextBatchIndex(rollout, int32(len(rollout.Spec.Strategy.GetSteps()))), + CurrentStepState: v1beta1.CanaryStepStateCompleted, + RolloutHash: rollout.Annotations[util.RolloutHashAnnotation], + }, + UpdatedRevision: workload.CanaryRevision, + } + } else { + newStatus.CanaryStatus = &v1beta1.CanaryStatus{ + CommonStatus: v1beta1.CommonStatus{ + ObservedRolloutID: getRolloutID(workload), + ObservedWorkloadGeneration: workload.Generation, + PodTemplateHash: workload.PodTemplateHash, + StableRevision: workload.StableRevision, + CurrentStepIndex: int32(len(rollout.Spec.Strategy.GetSteps())), + NextStepIndex: util.NextBatchIndex(rollout, int32(len(rollout.Spec.Strategy.GetSteps()))), + CurrentStepState: v1beta1.CanaryStepStateCompleted, + RolloutHash: rollout.Annotations[util.RolloutHashAnnotation], + }, + CanaryRevision: workload.CanaryRevision, + } } + newStatus.Message = "workload deployment is completed" } case v1beta1.RolloutPhaseDisabled: @@ -148,15 +167,30 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re // rolloutHash mainly records the step batch information, when the user step changes, // the current batch can be recalculated func (r *RolloutReconciler) calculateRolloutHash(rollout *v1beta1.Rollout) error { - canary := rollout.Spec.Strategy.Canary.DeepCopy() - canary.FailureThreshold = nil - canary.Steps = nil - for i := range rollout.Spec.Strategy.Canary.Steps { - step := rollout.Spec.Strategy.Canary.Steps[i].DeepCopy() - step.Pause = v1beta1.RolloutPause{} - canary.Steps = append(canary.Steps, *step) + var data string + if rollout.Spec.Strategy.IsCanaryStragegy() { + canary := rollout.Spec.Strategy.Canary.DeepCopy() + canary.FailureThreshold = nil + canary.Steps = nil + for i := range rollout.Spec.Strategy.Canary.Steps { + step := rollout.Spec.Strategy.Canary.Steps[i].DeepCopy() + step.Pause = v1beta1.RolloutPause{} + canary.Steps = append(canary.Steps, *step) + } + data = util.DumpJSON(canary) + } else if rollout.Spec.Strategy.IsBlueGreenRelease() { + blueGreen := rollout.Spec.Strategy.BlueGreen.DeepCopy() + blueGreen.FailureThreshold = nil + blueGreen.Steps = nil + for i := range rollout.Spec.Strategy.BlueGreen.Steps { + step := rollout.Spec.Strategy.BlueGreen.Steps[i].DeepCopy() + step.Pause = v1beta1.RolloutPause{} + blueGreen.Steps = append(blueGreen.Steps, *step) + } + data = util.DumpJSON(blueGreen) + } else { + return fmt.Errorf("unknown rolling style: %s", rollout.Spec.Strategy.GetRollingStyle()) } - data := util.DumpJSON(canary) hash := rand.SafeEncodeString(util.EncodeHash(data)) if rollout.Annotations[util.RolloutHashAnnotation] == hash { return nil diff --git a/pkg/trafficrouting/manager_test.go b/pkg/trafficrouting/manager_test.go index 366585e1..8ff7ce88 100644 --- a/pkg/trafficrouting/manager_test.go +++ b/pkg/trafficrouting/manager_test.go @@ -163,15 +163,17 @@ var ( Status: v1beta1.RolloutStatus{ Phase: v1beta1.RolloutPhaseProgressing, CanaryStatus: &v1beta1.CanaryStatus{ - ObservedWorkloadGeneration: 1, - RolloutHash: "rollout-hash-v1", - ObservedRolloutID: "rollout-id-1", - StableRevision: "podtemplatehash-v1", - CanaryRevision: "revision-v2", - CurrentStepIndex: 1, - CurrentStepState: v1beta1.CanaryStepStateTrafficRouting, - PodTemplateHash: "podtemplatehash-v2", - LastUpdateTime: &metav1.Time{Time: time.Now()}, + CommonStatus: v1beta1.CommonStatus{ + ObservedWorkloadGeneration: 1, + RolloutHash: "rollout-hash-v1", + ObservedRolloutID: "rollout-id-1", + StableRevision: "podtemplatehash-v1", + CurrentStepIndex: 1, + CurrentStepState: v1beta1.CanaryStepStateTrafficRouting, + PodTemplateHash: "podtemplatehash-v2", + LastUpdateTime: &metav1.Time{Time: time.Now()}, + }, + CanaryRevision: "revision-v2", }, Conditions: []v1beta1.RolloutCondition{ { @@ -249,15 +251,17 @@ var ( Status: v1beta1.RolloutStatus{ Phase: v1beta1.RolloutPhaseProgressing, CanaryStatus: &v1beta1.CanaryStatus{ - ObservedWorkloadGeneration: 1, - RolloutHash: "rollout-hash-v1", - ObservedRolloutID: "rollout-id-1", - StableRevision: "podtemplatehash-v1", - CanaryRevision: "revision-v2", - CurrentStepIndex: 1, - CurrentStepState: v1beta1.CanaryStepStateTrafficRouting, - PodTemplateHash: "podtemplatehash-v2", - LastUpdateTime: &metav1.Time{Time: time.Now()}, + CommonStatus: v1beta1.CommonStatus{ + ObservedWorkloadGeneration: 1, + RolloutHash: "rollout-hash-v1", + ObservedRolloutID: "rollout-id-1", + StableRevision: "podtemplatehash-v1", + CurrentStepIndex: 1, + CurrentStepState: v1beta1.CanaryStepStateTrafficRouting, + PodTemplateHash: "podtemplatehash-v2", + LastUpdateTime: &metav1.Time{Time: time.Now()}, + }, + CanaryRevision: "revision-v2", }, Conditions: []v1beta1.RolloutCondition{ { diff --git a/pkg/util/constant.go b/pkg/util/constant.go index 02ff7226..abfe2d98 100644 --- a/pkg/util/constant.go +++ b/pkg/util/constant.go @@ -27,7 +27,8 @@ const ( // BatchReleaseControlAnnotation is controller info about batchRelease when rollout BatchReleaseControlAnnotation = "batchrelease.rollouts.kruise.io/control-info" // InRolloutProgressingAnnotation marks workload as entering the rollout progressing process - //and does not allow paused=false during this process + // and does not allow paused=false during this process. However, blueGreen is an exception, + // which allows paused=false during progressing. InRolloutProgressingAnnotation = "rollouts.kruise.io/in-progressing" // RolloutHashAnnotation record observed rollout spec hash RolloutHashAnnotation = "rollouts.kruise.io/hash" diff --git a/pkg/util/controller_finder.go b/pkg/util/controller_finder.go index dc15ba1b..886b8191 100644 --- a/pkg/util/controller_finder.go +++ b/pkg/util/controller_finder.go @@ -87,7 +87,7 @@ func NewControllerFinder(c client.Client) *ControllerFinder { func (r *ControllerFinder) GetWorkloadForRef(rollout *rolloutv1beta1.Rollout) (*Workload, error) { workloadRef := rollout.Spec.WorkloadRef - if rollout.Spec.Strategy.Canary.EnableExtraWorkloadForCanary { + if rollout.Spec.Strategy.GetRollingStyle() == rolloutv1beta1.CanaryRollingStyle { for _, finder := range append(r.canaryStyleFinders(), r.partitionStyleFinders()...) { workload, err := finder(rollout.Namespace, &workloadRef) if workload != nil || err != nil { diff --git a/pkg/util/rollout_utils.go b/pkg/util/rollout_utils.go index f41d7bf7..33a2b0d9 100644 --- a/pkg/util/rollout_utils.go +++ b/pkg/util/rollout_utils.go @@ -47,7 +47,7 @@ type RolloutState struct { func IsRollbackInBatchPolicy(rollout *rolloutv1beta1.Rollout, labels map[string]string) bool { // currently, only support the case of no traffic routing - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if rollout.Spec.Strategy.HasTrafficRoutings() { return false } workloadRef := rollout.Spec.WorkloadRef @@ -164,3 +164,29 @@ func DumpJSON(o interface{}) string { func EncodeHash(data string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) } + +// calculate the next batch index +func NextBatchIndex(rollout *rolloutv1beta1.Rollout, CurrentStepIndex int32) int32 { + if rollout == nil { + return -1 + } + allSteps := int32(len(rollout.Spec.Strategy.GetSteps())) + if CurrentStepIndex >= allSteps { + return -1 + } + return CurrentStepIndex + 1 +} + +// check if NextStepIndex is legal, if not, correct it +func CheckNextBatchIndexWithCorrect(rollout *rolloutv1beta1.Rollout) { + if rollout == nil { + return + } + nextStep := rollout.Status.GetSubStatus().NextStepIndex + if nextStep <= 0 || nextStep > int32(len(rollout.Spec.Strategy.GetSteps())) { + rollout.Status.GetSubStatus().NextStepIndex = NextBatchIndex(rollout, rollout.Status.GetSubStatus().CurrentStepIndex) + if nextStep != rollout.Status.GetSubStatus().NextStepIndex { + klog.Infof("rollout(%s/%s) invalid nextStepIndex(%d), reset to %d", rollout.Namespace, rollout.Name, nextStep, rollout.Status.GetSubStatus().NextStepIndex) + } + } +} diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index 0d2c78b9..43db8927 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -133,12 +133,11 @@ func (h *RolloutCreateUpdateHandler) validateRolloutUpdate(oldObj, newObj *appsv if !reflect.DeepEqual(oldObj.Spec.WorkloadRef, newObj.Spec.WorkloadRef) { return field.ErrorList{field.Forbidden(field.NewPath("Spec.ObjectRef"), "Rollout 'ObjectRef' field is immutable")} } - // canary strategy - if !reflect.DeepEqual(oldObj.Spec.Strategy.Canary.TrafficRoutings, newObj.Spec.Strategy.Canary.TrafficRoutings) { - return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy.Canary.TrafficRoutings"), "Rollout 'Strategy.Canary.TrafficRoutings' field is immutable")} + if !reflect.DeepEqual(oldObj.Spec.Strategy.GetTrafficRouting(), newObj.Spec.Strategy.GetTrafficRouting()) { + return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy.Canary|BlueGreen.TrafficRoutings"), "Rollout 'Strategy.Canary|BlueGreen.TrafficRoutings' field is immutable")} } - if oldObj.Spec.Strategy.Canary.EnableExtraWorkloadForCanary != newObj.Spec.Strategy.Canary.EnableExtraWorkloadForCanary { - return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy.Canary"), "Rollout enableExtraWorkloadForCanary is immutable")} + if oldObj.Spec.Strategy.GetRollingStyle() != newObj.Spec.Strategy.GetRollingStyle() { + return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy.Canary|BlueGreen"), "Rollout style and enableExtraWorkloadForCanary are immutable")} } } @@ -198,15 +197,32 @@ func validateRolloutSpecObjectRef(workloadRef *appsv1beta1.ObjectRef, fldPath *f } func validateRolloutSpecStrategy(strategy *appsv1beta1.RolloutStrategy, fldPath *field.Path) field.ErrorList { + if strategy.Canary == nil && strategy.BlueGreen == nil { + return field.ErrorList{field.Invalid(fldPath, nil, "Canary and BlueGreen cannot both be empty")} + } + if strategy.Canary != nil && strategy.BlueGreen != nil { + return field.ErrorList{field.Invalid(fldPath, nil, "Canary and BlueGreen cannot both be set")} + } + if strategy.BlueGreen != nil { + return validateRolloutSpecBlueGreenStrategy(strategy.BlueGreen, fldPath.Child("BlueGreen")) + } return validateRolloutSpecCanaryStrategy(strategy.Canary, fldPath.Child("Canary")) } +type TrafficRule string + +const ( + TrafficRuleCanary TrafficRule = "Canary" + TrafficRuleBlueGreen TrafficRule = "BlueGreen" + NoTraffic TrafficRule = "NoTraffic" +) + func validateRolloutSpecCanaryStrategy(canary *appsv1beta1.CanaryStrategy, fldPath *field.Path) field.ErrorList { - if canary == nil { - return field.ErrorList{field.Invalid(fldPath, nil, "Canary cannot be empty")} + trafficRule := NoTraffic + if len(canary.TrafficRoutings) > 0 { + trafficRule = TrafficRuleCanary } - - errList := validateRolloutSpecCanarySteps(canary.Steps, fldPath.Child("Steps"), len(canary.TrafficRoutings) > 0) + errList := validateRolloutSpecCanarySteps(canary.Steps, fldPath.Child("Steps"), trafficRule) if len(canary.TrafficRoutings) > 1 { errList = append(errList, field.Invalid(fldPath, canary.TrafficRoutings, "Rollout currently only support single TrafficRouting.")) } @@ -216,6 +232,21 @@ func validateRolloutSpecCanaryStrategy(canary *appsv1beta1.CanaryStrategy, fldPa return errList } +func validateRolloutSpecBlueGreenStrategy(blueGreen *appsv1beta1.BlueGreenStrategy, fldPath *field.Path) field.ErrorList { + trafficRule := NoTraffic + if len(blueGreen.TrafficRoutings) > 0 { + trafficRule = TrafficRuleBlueGreen + } + errList := validateRolloutSpecCanarySteps(blueGreen.Steps, fldPath.Child("Steps"), trafficRule) + if len(blueGreen.TrafficRoutings) > 1 { + errList = append(errList, field.Invalid(fldPath, blueGreen.TrafficRoutings, "Rollout currently only support single TrafficRouting.")) + } + for _, traffic := range blueGreen.TrafficRoutings { + errList = append(errList, validateRolloutSpecCanaryTraffic(traffic, fldPath.Child("TrafficRouting"))...) + } + return errList +} + func validateRolloutSpecCanaryTraffic(traffic appsv1beta1.TrafficRoutingRef, fldPath *field.Path) field.ErrorList { errList := field.ErrorList{} if len(traffic.Service) == 0 { @@ -240,7 +271,7 @@ func validateRolloutSpecCanaryTraffic(traffic appsv1beta1.TrafficRoutingRef, fld return errList } -func validateRolloutSpecCanarySteps(steps []appsv1beta1.CanaryStep, fldPath *field.Path, isTraffic bool) field.ErrorList { +func validateRolloutSpecCanarySteps(steps []appsv1beta1.CanaryStep, fldPath *field.Path, trafficRule TrafficRule) field.ErrorList { stepCount := len(steps) if stepCount == 0 { return field.ErrorList{field.Invalid(fldPath, steps, "The number of Canary.Steps cannot be empty")} @@ -258,14 +289,21 @@ func validateRolloutSpecCanarySteps(steps []appsv1beta1.CanaryStep, fldPath *fie return field.ErrorList{field.Invalid(fldPath.Index(i).Child("Replicas"), s.Replicas, `replicas must be positive number, or a percentage with "0%" < canaryReplicas <= "100%"`)} } - if !isTraffic { + if trafficRule == NoTraffic || s.Traffic == nil { continue } - if s.Traffic != nil { - is := intstr.FromString(*s.Traffic) - weight, err := intstr.GetScaledValueFromIntOrPercent(&is, 100, true) + is := intstr.FromString(*s.Traffic) + weight, err := intstr.GetScaledValueFromIntOrPercent(&is, 100, true) + switch trafficRule { + case TrafficRuleBlueGreen: + // traffic "0%" is allowed in blueGreen strategy + if err != nil || weight < 0 || weight > 100 { + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `traffic must be percentage with "0%" <= traffic <= "100%" in blueGreen strategy`)} + } + default: + // traffic "0%" is not allowed in canary strategy if err != nil || weight <= 0 || weight > 100 { - return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `traffic must be percentage with "0%" < traffic <= "100%"`)} + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `traffic must be percentage with "0%" < traffic <= "100%" in canary strategy`)} } } } diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go index 07eacd57..27ea4531 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go @@ -96,7 +96,9 @@ var ( }, Status: appsv1beta1.RolloutStatus{ CanaryStatus: &appsv1beta1.CanaryStatus{ - CurrentStepState: appsv1beta1.CanaryStepStateCompleted, + CommonStatus: appsv1beta1.CommonStatus{ + CurrentStepState: appsv1beta1.CanaryStepStateCompleted, + }, }, }, } diff --git a/pkg/webhook/workload/mutating/workload_update_handler.go b/pkg/webhook/workload/mutating/workload_update_handler.go index 60a443de..45ade337 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler.go +++ b/pkg/webhook/workload/mutating/workload_update_handler.go @@ -242,7 +242,7 @@ func (h *WorkloadHandler) handleStatefulSetLikeWorkload(newObj, oldObj *unstruct rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } @@ -310,7 +310,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } rss, err := h.Finder.GetReplicaSetsForDeployment(newObj) @@ -319,7 +319,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo return false, nil } // if traffic routing, workload must only be one version of Pods - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if rollout.Spec.Strategy.HasTrafficRoutings() { if len(rss) != 1 { klog.Warningf("Because deployment(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) return false, nil @@ -334,6 +334,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo if newObj.Labels == nil { newObj.Labels = map[string]string{} } + // blueGreen also need the stable revision label newObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = stableRS.Labels[apps.DefaultDeploymentUniqueLabelKey] } @@ -365,11 +366,11 @@ func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.Clon rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } // if traffic routing, there must only be one version of Pods - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { + if rollout.Spec.Strategy.HasTrafficRoutings() && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { klog.Warningf("Because cloneSet(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) return false, nil } @@ -398,7 +399,7 @@ func (h *WorkloadHandler) handleDaemonSet(newObj, oldObj *kruiseappsv1alpha1.Dae rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } diff --git a/test/e2e/deployment_test.go b/test/e2e/deployment_test.go index b857f929..c8379bc4 100644 --- a/test/e2e/deployment_test.go +++ b/test/e2e/deployment_test.go @@ -26,6 +26,16 @@ import ( var _ = SIGDescribe("Advanced Deployment", func() { var namespace string + + DumpAllResources := func() { + deploy := &apps.DeploymentList{} + k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(deploy)) + rs := &apps.ReplicaSetList{} + k8sClient.List(context.TODO(), rs, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rs)) + } + defaultRetry := wait.Backoff{ Steps: 10, Duration: 10 * time.Millisecond, @@ -132,7 +142,12 @@ var _ = SIGDescribe("Advanced Deployment", func() { CheckReplicas := func(deployment *apps.Deployment, replicas, available, updated int32) { var clone *apps.Deployment + start := time.Now() Eventually(func() bool { + if start.Add(time.Minute * 2).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } clone = &apps.Deployment{} err := GetObject(deployment.Namespace, deployment.Name, clone) Expect(err).NotTo(HaveOccurred()) @@ -239,6 +254,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { deployment.Namespace = namespace Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithCheck(deployment, intstr.FromInt(1)) @@ -255,6 +271,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) deployment.Spec.Replicas = pointer.Int32(10) CreateObject(deployment) + CheckReplicas(deployment, 10, 10, 10) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromString("0%")) UpdatePartitionWithCheck(deployment, intstr.FromString("40%")) @@ -287,6 +304,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { `{"rollingStyle":"Partition","rollingUpdate":{"maxUnavailable":1,"maxSurge":0}}` deployment.Spec.MinReadySeconds = 10 CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithoutCheck(deployment, intstr.FromInt(3)) @@ -303,6 +321,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { deployment.Namespace = namespace Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithCheck(deployment, intstr.FromInt(2)) @@ -317,6 +336,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { deployment.Namespace = namespace Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithCheck(deployment, intstr.FromInt(2)) @@ -335,6 +355,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) deployment.Annotations["rollouts.kruise.io/deployment-strategy"] = `{"rollingUpdate":{"maxUnavailable":0,"maxSurge":1}}` CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2", "busybox:not-exists") UpdatePartitionWithoutCheck(deployment, intstr.FromInt(1)) CheckReplicas(deployment, 6, 5, 1) diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index c8a7cae2..c2ae86c8 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -204,18 +204,25 @@ var _ = SIGDescribe("Rollout", func() { } ResumeRolloutCanary := func(name string) { + clone := &v1alpha1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + currentIndex := clone.Status.CanaryStatus.CurrentStepIndex Eventually(func() bool { clone := &v1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) - if clone.Status.CanaryStatus.CurrentStepState != v1alpha1.CanaryStepStatePaused { + if clone.Status.CanaryStatus.CurrentStepIndex == currentIndex && clone.Status.CanaryStatus.CurrentStepState == v1alpha1.CanaryStepStatePaused { + klog.Info("patch to stepReady") + body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1alpha1.CanaryStepStateReady) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + } else { fmt.Println("resume rollout success, and CurrentStepState", util.DumpJSON(clone.Status)) return true } - - body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1alpha1.CanaryStepStateReady) - Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) - return false - }, 10*time.Second, time.Second).Should(BeTrue()) + // interval was critical before: + // too small: StepReady could be overidden by StepPaused + // too big: StepReady could progress to StepPaused of next Step + }, 10*time.Second, 2*time.Second).Should(BeTrue()) } WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) { diff --git a/test/e2e/rollout_v1beta1_test.go b/test/e2e/rollout_v1beta1_test.go new file mode 100644 index 00000000..31b1c1dc --- /dev/null +++ b/test/e2e/rollout_v1beta1_test.go @@ -0,0 +1,1634 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + appsv1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + "github.com/openkruise/rollouts/api/v1beta1" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + // "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + // "k8s.io/apimachinery/pkg/util/intstr" + // gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + // "github.com/openkruise/rollouts/api/v1alpha1" + // "k8s.io/apimachinery/pkg/api/errors" +) + +var _ = SIGDescribe("Rollout v1beta1", func() { + var namespace string + + DumpAllResources := func() { + rollout := &v1beta1.RolloutList{} + k8sClient.List(context.TODO(), rollout, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rollout)) + batch := &v1beta1.BatchReleaseList{} + k8sClient.List(context.TODO(), batch, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(batch)) + deploy := &apps.DeploymentList{} + k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(deploy)) + rs := &apps.ReplicaSetList{} + k8sClient.List(context.TODO(), rs, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rs)) + cloneSet := &appsv1alpha1.CloneSetList{} + k8sClient.List(context.TODO(), cloneSet, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(cloneSet)) + sts := &apps.StatefulSetList{} + k8sClient.List(context.TODO(), sts, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(sts)) + asts := &appsv1beta1.StatefulSetList{} + k8sClient.List(context.TODO(), asts, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(asts)) + } + + CreateObject := func(object client.Object, options ...client.CreateOption) { + object.SetNamespace(namespace) + Expect(k8sClient.Create(context.TODO(), object)).NotTo(HaveOccurred()) + } + + GetObject := func(name string, object client.Object) error { + key := types.NamespacedName{Namespace: namespace, Name: name} + return k8sClient.Get(context.TODO(), key, object) + } + + getRolloutCondition := func(status v1beta1.RolloutStatus, condType v1beta1.RolloutConditionType) *v1beta1.RolloutCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil + } + + UpdateDeployment := func(object *apps.Deployment) *apps.Deployment { + var clone *apps.Deployment + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &apps.Deployment{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + clone.Spec.Paused = object.Spec.Paused + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + UpdateCloneSet := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet { + var clone *appsv1alpha1.CloneSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &appsv1alpha1.CloneSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + // UpdateDaemonSet := func(object *appsv1alpha1.DaemonSet) *appsv1alpha1.DaemonSet { + // var daemon *appsv1alpha1.DaemonSet + // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + // daemon = &appsv1alpha1.DaemonSet{} + // err := GetObject(object.Name, daemon) + // if err != nil { + // return err + // } + // // daemon.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + // daemon.Spec.Template = *object.Spec.Template.DeepCopy() + // daemon.Spec.UpdateStrategy = *object.Spec.UpdateStrategy.DeepCopy() + // daemon.Labels = mergeMap(daemon.Labels, object.Labels) + // daemon.Annotations = mergeMap(daemon.Annotations, object.Annotations) + // return k8sClient.Update(context.TODO(), daemon) + // })).NotTo(HaveOccurred()) + + // return daemon + // } + + // UpdateNativeStatefulSet := func(object *apps.StatefulSet) *apps.StatefulSet { + // var clone *apps.StatefulSet + // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + // clone = &apps.StatefulSet{} + // err := GetObject(object.Name, clone) + // if err != nil { + // return err + // } + // clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + // clone.Spec.Template = *object.Spec.Template.DeepCopy() + // clone.Labels = mergeMap(clone.Labels, object.Labels) + // clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + // return k8sClient.Update(context.TODO(), clone) + // })).NotTo(HaveOccurred()) + + // return clone + // } + + // UpdateAdvancedStatefulSet := func(object *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { + // var clone *appsv1beta1.StatefulSet + // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + // clone = &appsv1beta1.StatefulSet{} + // err := GetObject(object.Name, clone) + // if err != nil { + // return err + // } + // clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + // clone.Spec.Template = *object.Spec.Template.DeepCopy() + // clone.Labels = mergeMap(clone.Labels, object.Labels) + // clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + // return k8sClient.Update(context.TODO(), clone) + // })).NotTo(HaveOccurred()) + + // return clone + // } + + UpdateRollout := func(object *v1beta1.Rollout) *v1beta1.Rollout { + var clone *v1beta1.Rollout + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &v1beta1.Rollout{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec = *object.Spec.DeepCopy() + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + ResumeRolloutCanary := func(name string) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus.CurrentStepState != v1beta1.CanaryStepStatePaused { + fmt.Println("resume rollout success, and CurrentStepState", util.DumpJSON(clone.Status)) + return true + } + + body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1beta1.CanaryStepStateReady) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + }, 10*time.Second, time.Second).Should(BeTrue()) + } + + RolloutJumpCanaryStep := func(name string, target int) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus.CurrentStepState != v1beta1.CanaryStepStatePaused { + fmt.Println("Jump successfully, and current status ", util.DumpJSON(clone.Status)) + return true + } + + body := fmt.Sprintf(`{"status":{"canaryStatus":{"nextStepIndex":%d}}}`, target) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + }, 10*time.Second, time.Second).Should(BeTrue()) + } + + // RolloutJumpBlueGreenStep := func(name string, target int) { + // Eventually(func() bool { + // clone := &v1alpha1.Rollout{} + // Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + // if clone.Status.CanaryStatus.CurrentStepState !=v1beta1.CanaryStepStatePaused { + // fmt.Println("Jump successfully, and current status ", util.DumpJSON(clone.Status)) + // return true + // } + + // body := fmt.Sprintf(`{"status":{"blueGreenStatus":{"nextStepIndex":"%d"}}}`, target) + // Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + // return false + // }, 10*time.Second, time.Second).Should(BeTrue()) + // } + + WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) { + Eventually(func() bool { + clone := &apps.Deployment{} + Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && *clone.Spec.Replicas == clone.Status.UpdatedReplicas && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 5*time.Minute, time.Second).Should(BeTrue()) + } + + WaitCloneSetAllPodsReady := func(cloneset *appsv1alpha1.CloneSet) { + Eventually(func() bool { + clone := &appsv1alpha1.CloneSet{} + Expect(GetObject(cloneset.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && *clone.Spec.Replicas == clone.Status.UpdatedReplicas && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 5*time.Minute, time.Second).Should(BeTrue()) + } + + // WaitNativeStatefulSetPodsReady := func(statefulset *apps.StatefulSet) { + // Eventually(func() bool { + // set := &apps.StatefulSet{} + // Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + // return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + // *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + // }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + // } + + // WaitAdvancedStatefulSetPodsReady := func(statefulset *appsv1beta1.StatefulSet) { + // Eventually(func() bool { + // set := &appsv1beta1.StatefulSet{} + // Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + // return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + // *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + // }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + // } + + // WaitDaemonSetAllPodsReady := func(daemonset *appsv1alpha1.DaemonSet) { + // Eventually(func() bool { + // daemon := &appsv1alpha1.DaemonSet{} + // Expect(GetObject(daemonset.Name, daemon)).NotTo(HaveOccurred()) + // klog.Infof("DaemonSet updateStrategy(%s) Generation(%d) ObservedGeneration(%d) DesiredNumberScheduled(%d) UpdatedNumberScheduled(%d) NumberReady(%d)", + // util.DumpJSON(daemon.Spec.UpdateStrategy), daemon.Generation, daemon.Status.ObservedGeneration, daemon.Status.DesiredNumberScheduled, daemon.Status.UpdatedNumberScheduled, daemon.Status.NumberReady) + // return daemon.Status.ObservedGeneration == daemon.Generation && daemon.Status.DesiredNumberScheduled == daemon.Status.UpdatedNumberScheduled && daemon.Status.DesiredNumberScheduled == daemon.Status.NumberReady + // }, 5*time.Minute, time.Second).Should(BeTrue()) + // } + + // WaitDeploymentReplicas := func(deployment *apps.Deployment) { + // Eventually(func() bool { + // clone := &apps.Deployment{} + // Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) + // return clone.Status.ObservedGeneration == clone.Generation && + // *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + // }, 10*time.Minute, time.Second).Should(BeTrue()) + // } + + WaitRolloutCanaryStepPaused := func(name string, stepIndex int32) { + start := time.Now() + Eventually(func() bool { + if start.Add(time.Minute * 5).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus == nil { + return false + } + klog.Infof("current step:%v target step:%v current step state %v", clone.Status.CanaryStatus.CurrentStepIndex, stepIndex, clone.Status.CanaryStatus.CurrentStepState) + return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == v1beta1.CanaryStepStatePaused + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + WaitRolloutStatusPhase := func(name string, phase v1beta1.RolloutPhase) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + return clone.Status.Phase == phase + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + WaitRolloutWorkloadGeneration := func(name string, generation int64) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + return clone.Status.CanaryStatus.ObservedWorkloadGeneration == generation + }, time.Minute, time.Second).Should(BeTrue()) + } + + // WaitRolloutNotFound := func(name string) { + // Eventually(func() bool { + // clone := &v1beta1.Rollout{} + // err := GetObject(name, clone) + // if err == nil { + // return false + // } else if errors.IsNotFound(err) { + // return true + // } else { + // Expect(err).NotTo(HaveOccurred()) + // return false + // } + // }, 5*time.Minute, time.Second).Should(BeTrue()) + // } + + GetCanaryDeployment := func(stable *apps.Deployment) (*apps.Deployment, error) { + canaryList := &apps.DeploymentList{} + selector, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.CanaryDeploymentLabel: stable.Name}}) + err := k8sClient.List(context.TODO(), canaryList, &client.ListOptions{Namespace: stable.Namespace, LabelSelector: selector}) + if err != nil { + return nil, err + } else if len(canaryList.Items) == 0 { + return nil, nil + } + sort.Slice(canaryList.Items, func(i, j int) bool { + return canaryList.Items[j].CreationTimestamp.Before(&canaryList.Items[i].CreationTimestamp) + }) + return &canaryList.Items[0], nil + } + + // ListPods := func(namespace string, labelSelector *metav1.LabelSelector) ([]*v1.Pod, error) { + // appList := &v1.PodList{} + // selector, _ := metav1.LabelSelectorAsSelector(labelSelector) + // err := k8sClient.List(context.TODO(), appList, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) + // if err != nil { + // return nil, err + // } + // apps := make([]*v1.Pod, 0) + // for i := range appList.Items { + // pod := &appList.Items[i] + // if pod.DeletionTimestamp.IsZero() { + // apps = append(apps, pod) + // } + // } + // return apps, nil + // } + + // CheckPodBatchLabel := func(namespace string, labelSelector *metav1.LabelSelector, rolloutID, batchID string, expected int) { + // pods, err := ListPods(namespace, labelSelector) + // Expect(err).NotTo(HaveOccurred()) + + // count := 0 + // for _, pod := range pods { + // if pod.Labels[v1beta1.RolloutIDLabel] == rolloutID && + // pod.Labels[v1beta1.RolloutBatchIDLabel] == batchID { + // count++ + // } + // } + // Expect(count).Should(BeNumerically("==", expected)) + // } + + ListReplicaSet := func(d *apps.Deployment) []*apps.ReplicaSet { + var rss []*apps.ReplicaSet + rsLister := &apps.ReplicaSetList{} + selectorOpt, _ := metav1.LabelSelectorAsSelector(d.Spec.Selector) + err := k8sClient.List(context.TODO(), rsLister, &client.ListOptions{LabelSelector: selectorOpt, Namespace: d.Namespace}) + Expect(err).NotTo(HaveOccurred()) + for i := range rsLister.Items { + rs := &rsLister.Items[i] + if !rs.DeletionTimestamp.IsZero() { + continue + } + rss = append(rss, rs) + } + return rss + } + + GetStableRSRevision := func(d *apps.Deployment) string { + rss := ListReplicaSet(d) + _, stable := util.FindCanaryAndStableReplicaSet(rss, d) + if stable != nil { + return stable.Labels[apps.DefaultDeploymentUniqueLabelKey] + } + return "" + } + + GetCanaryRSRevision := func(d *apps.Deployment) string { + rss := ListReplicaSet(d) + canary, _ := util.FindCanaryAndStableReplicaSet(rss, d) + if canary != nil { + return canary.Labels[apps.DefaultDeploymentUniqueLabelKey] + } + return "" + } + + BeforeEach(func() { + namespace = randomNamespaceName("rollout") + ns := v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + Expect(k8sClient.Create(context.TODO(), &ns)).Should(SatisfyAny(BeNil())) + }) + + AfterEach(func() { + By("[TEST] Clean up resources after an integration test") + k8sClient.DeleteAllOf(context.TODO(), &apps.Deployment{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &appsv1alpha1.CloneSet{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1beta1.BatchRelease{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1beta1.Rollout{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1.Service{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &netv1.Ingress{}, client.InNamespace(namespace)) + Expect(k8sClient.Delete(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) + time.Sleep(time.Second * 3) + }) + + KruiseDescribe("Step Jump", func() { + // step1-> 2-> 3-> 4-> 3-(TrafficChange)-> 3-> 2-> 1-> 5 + It("V1->V2: Deployment, Canary, patch nextStepIndex to jump", func() { + finder := util.NewControllerFinder(k8sClient) + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update deployment image from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + // canary workload + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 1)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + + // wait step 2 complete + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + // canary workload + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 2)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + + // wait step 3 complete + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + // rollout + WaitRolloutCanaryStepPaused(rollout.Name, 3) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + // canary workload + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 3)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + + // wait step 4 complete + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) + // canary workload + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + // canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + + // Jump to step 3 + By("Jump to step 3") + RolloutJumpCanaryStep(rollout.Name, 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("21%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("41%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("61%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("81%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("100%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Jump to step 2 + By("Jump to step 2") + RolloutJumpCanaryStep(rollout.Name, 2) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // Jump to step 1 + By("Jump to step 1") + RolloutJumpCanaryStep(rollout.Name, 1) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // Jump to step 5 + By("Jump to step 5") + RolloutJumpCanaryStep(rollout.Name, 5) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + klog.Infof("rollout(%s) completed, and check", namespace) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) + // check service & ingress & deployment + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // deployment + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Paused).Should(BeFalse()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + + }) + + // step1-> 2-> 3-> 4-> 3-(TrafficChange)-> 3-> 2-> 1-> 5 + It("V1->V2: Deployment, Partition, patch nextStepIndex to jump", func() { + finder := util.NewControllerFinder(k8sClient) + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update deployment image from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // wait step 2 complete + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 2)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 2)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // wait step 3 complete + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // wait step 4 complete + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + + // Jump to step 3 + By("Jump to step 3") + RolloutJumpCanaryStep(rollout.Name, 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + // won't scale down + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("21%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("41%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("61%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("81%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("100%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + // won't scale down + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Jump to step 2 + By("Jump to step 2") + RolloutJumpCanaryStep(rollout.Name, 2) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // Jump to step 1 + By("Jump to step 1") + RolloutJumpCanaryStep(rollout.Name, 1) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // Jump to step 5 + By("Jump to step 5") + RolloutJumpCanaryStep(rollout.Name, 5) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhase(v1beta1.RolloutPhaseHealthy)) + klog.Infof("rollout(%s) completed, and check", namespace) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) + // check service & ingress & deployment + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // deployment + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Paused).Should(BeFalse()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + // step1-> 2-> 3-> 4-> 3-(TrafficChange)-> 3-> 2-> 1-> 5 + It("V1->V2: CloneSet, Partition, patch nextStepIndex to jump", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // wait step 2 complete + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 2)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 2)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // wait step 3 complete + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("21%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("41%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("61%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("81%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("100%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // wait step 4 complete + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + + // Jump to step 3 + By("Jump to step 3") + RolloutJumpCanaryStep(rollout.Name, 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Jump to step 2 + By("Jump to step 2") + RolloutJumpCanaryStep(rollout.Name, 2) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // Jump to step 1 + By("Jump to step 1") + RolloutJumpCanaryStep(rollout.Name, 1) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // Jump to step 5 + By("Jump to step 5") + RolloutJumpCanaryStep(rollout.Name, 5) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhase(v1beta1.RolloutPhaseHealthy)) + klog.Infof("rollout(%s) completed, and check", namespace) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) + // check service & ingress + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // clonese + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + + }) + }) +}) + +func removePercentageSign(input string) string { + if input == "0" { + return "0" + } + if strings.HasSuffix(input, "%") { + return strings.TrimSuffix(input, "%") + } + fmt.Printf("input(%s) has no percentage sign!", input) + return "" +} diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_bluegreen_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_bluegreen_base.yaml new file mode 100644 index 00000000..0959f065 --- /dev/null +++ b/test/e2e/test_data/rollout/rollout_v1beta1_bluegreen_base.yaml @@ -0,0 +1,32 @@ +apiVersion: rollouts.kruise.io/v1beta1 # we use v1beta1 +kind: Rollout +metadata: + name: rollouts-demo +spec: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: echoserver + strategy: + blueGreen: + steps: + - traffic: 20% + replicas: 20% + pause: {} + - traffic: 40% + replicas: 40% + pause: {duration: 10} + - traffic: 60% + replicas: 60% + pause: {duration: 10} + - traffic: 80% + replicas: 80% + pause: {duration: 10} + - traffic: 100% + replicas: 100% + pause: {duration: 0} + trafficRoutings: + - service: echoserver + ingress: + classType: nginx + name: echoserver diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_canary_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_canary_base.yaml new file mode 100644 index 00000000..d3b2cbc6 --- /dev/null +++ b/test/e2e/test_data/rollout/rollout_v1beta1_canary_base.yaml @@ -0,0 +1,33 @@ +apiVersion: rollouts.kruise.io/v1beta1 # we use v1beta1 +kind: Rollout +metadata: + name: rollouts-demo +spec: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: echoserver + strategy: + canary: + enableExtraWorkloadForCanary: true + steps: + - traffic: 20% + replicas: 20% + pause: {} + - traffic: 40% + replicas: 40% + pause: {duration: 10} + - traffic: 60% + replicas: 60% + pause: {duration: 10} + - traffic: 80% + replicas: 80% + pause: {duration: 10} + - traffic: 100% + replicas: 100% + pause: {duration: 0} + trafficRoutings: + - service: echoserver + ingress: + classType: nginx + name: echoserver diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml new file mode 100644 index 00000000..2b492ea7 --- /dev/null +++ b/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml @@ -0,0 +1,33 @@ +apiVersion: rollouts.kruise.io/v1beta1 # we use v1beta1 +kind: Rollout +metadata: + name: rollouts-demo +spec: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: echoserver + strategy: + canary: + enableExtraWorkloadForCanary: false + steps: + - traffic: 20% + replicas: 20% + pause: {} + - traffic: 40% + replicas: 40% + pause: {duration: 10} + - traffic: 60% + replicas: 60% + pause: {duration: 10} + - traffic: 80% + replicas: 80% + pause: {duration: 10} + - traffic: 100% + replicas: 100% + pause: {duration: 0} + trafficRoutings: + - service: echoserver + ingress: + classType: nginx + name: echoserver