diff --git a/manifests/crds/rollout-crd.yaml b/manifests/crds/rollout-crd.yaml index 13e8b1b718..0e85a68c51 100644 --- a/manifests/crds/rollout-crd.yaml +++ b/manifests/crds/rollout-crd.yaml @@ -64,8 +64,6 @@ spec: revisionHistoryLimit: format: int32 type: integer - scaleDownOnAbort: - type: boolean selector: properties: matchExpressions: @@ -93,6 +91,9 @@ spec: properties: blueGreen: properties: + abortScaleDownDelaySeconds: + format: int32 + type: integer activeMetadata: properties: annotations: @@ -226,6 +227,9 @@ spec: type: object canary: properties: + abortScaleDownDelaySeconds: + format: int32 + type: integer analysis: properties: args: diff --git a/manifests/install.yaml b/manifests/install.yaml index 7fd6e77cfb..278bded306 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -9753,8 +9753,6 @@ spec: revisionHistoryLimit: format: int32 type: integer - scaleDownOnAbort: - type: boolean selector: properties: matchExpressions: @@ -9782,6 +9780,9 @@ spec: properties: blueGreen: properties: + abortScaleDownDelaySeconds: + format: int32 + type: integer activeMetadata: properties: annotations: @@ -9915,6 +9916,9 @@ spec: type: object canary: properties: + abortScaleDownDelaySeconds: + format: int32 + type: integer analysis: properties: args: diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml index 666fe9d70b..e944acb4f2 100644 --- a/manifests/namespace-install.yaml +++ b/manifests/namespace-install.yaml @@ -9753,8 +9753,6 @@ spec: revisionHistoryLimit: format: int32 type: integer - scaleDownOnAbort: - type: boolean selector: properties: matchExpressions: @@ -9782,6 +9780,9 @@ spec: properties: blueGreen: properties: + abortScaleDownDelaySeconds: + format: int32 + type: integer activeMetadata: properties: annotations: @@ -9915,6 +9916,9 @@ spec: type: object canary: properties: + abortScaleDownDelaySeconds: + format: int32 + type: integer analysis: properties: args: diff --git a/pkg/apiclient/rollout/rollout.swagger.json b/pkg/apiclient/rollout/rollout.swagger.json index 4521a8f6d8..eb96ecb356 100644 --- a/pkg/apiclient/rollout/rollout.swagger.json +++ b/pkg/apiclient/rollout/rollout.swagger.json @@ -662,6 +662,11 @@ "activeMetadata": { "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PodTemplateMetadata", "title": "ActiveMetadata specify labels and annotations which will be attached to the active pods for\nthe duration which they act as a active pod, and will be removed after" + }, + "abortScaleDownDelaySeconds": { + "type": "integer", + "format": "int32", + "title": "AbortScaleDownDelaySeconds\n+optional" } }, "title": "BlueGreenStrategy defines parameters for Blue Green deployment" @@ -766,6 +771,11 @@ "type": "integer", "format": "int32", "title": "ScaleDownDelayRevisionLimit limits the number of old RS that can run at one time before getting scaled down\n+optional" + }, + "abortScaleDownDelaySeconds": { + "type": "integer", + "format": "int32", + "title": "AbortScaleDownDelaySeconds speficifes\n+optional" } }, "title": "CanaryStrategy defines parameters for a Replica Based Canary" @@ -1155,10 +1165,6 @@ "restartAt": { "$ref": "#/definitions/k8s.io.apimachinery.pkg.apis.meta.v1.Time", "title": "RestartAt indicates when all the pods of a Rollout should be restarted" - }, - "scaleDownOnAbort": { - "type": "boolean", - "title": "ScaleDownOnAbort scales down" } }, "title": "RolloutSpec is the spec for a Rollout resource" diff --git a/pkg/apis/rollouts/v1alpha1/types.go b/pkg/apis/rollouts/v1alpha1/types.go index 79417bbd34..943feb4e9a 100644 --- a/pkg/apis/rollouts/v1alpha1/types.go +++ b/pkg/apis/rollouts/v1alpha1/types.go @@ -71,8 +71,6 @@ type RolloutSpec struct { ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty" protobuf:"varint,8,opt,name=progressDeadlineSeconds"` // RestartAt indicates when all the pods of a Rollout should be restarted RestartAt *metav1.Time `json:"restartAt,omitempty" protobuf:"bytes,9,opt,name=restartAt"` - // ScaleDownOnAbort scales down - ScaleDownOnAbort bool `json:"scaleDownOnAbort,omitempty" protobuf:"varint,11,opt,name=scaleDownOnAbort"` } func (s *RolloutSpec) SetResolvedSelector(selector *metav1.LabelSelector) { @@ -194,6 +192,9 @@ type BlueGreenStrategy struct { // ActiveMetadata specify labels and annotations which will be attached to the active pods for // the duration which they act as a active pod, and will be removed after ActiveMetadata *PodTemplateMetadata `json:"activeMetadata,omitempty" protobuf:"bytes,13,opt,name=activeMetadata"` + // AbortScaleDownDelaySeconds + // +optional + AbortScaleDownDelaySeconds *int32 `json:"abortScaleDownDelaySeconds,omitempty" protobuf:"varint,14,opt,name=abortScaleDownDelaySeconds"` } // AntiAffinity defines which inter-pod scheduling rule to use for anti-affinity injection @@ -274,6 +275,9 @@ type CanaryStrategy struct { // ScaleDownDelayRevisionLimit limits the number of old RS that can run at one time before getting scaled down // +optional ScaleDownDelayRevisionLimit *int32 `json:"scaleDownDelayRevisionLimit,omitempty" protobuf:"varint,12,opt,name=scaleDownDelayRevisionLimit"` + // AbortScaleDownDelaySeconds speficifes + // +optional + AbortScaleDownDelaySeconds *int32 `json:"abortScaleDownDelaySeconds,omitempty" protobuf:"varint,13,opt,name=abortScaleDownDelaySeconds"` } // ALBTrafficRouting configuration for ALB ingress controller to control traffic routing diff --git a/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go index ef2cd8be0c..f064a2e228 100644 --- a/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go @@ -449,6 +449,11 @@ func (in *BlueGreenStrategy) DeepCopyInto(out *BlueGreenStrategy) { *out = new(PodTemplateMetadata) (*in).DeepCopyInto(*out) } + if in.AbortScaleDownDelaySeconds != nil { + in, out := &in.AbortScaleDownDelaySeconds, &out.AbortScaleDownDelaySeconds + *out = new(int32) + **out = **in + } return } @@ -584,6 +589,11 @@ func (in *CanaryStrategy) DeepCopyInto(out *CanaryStrategy) { *out = new(int32) **out = **in } + if in.AbortScaleDownDelaySeconds != nil { + in, out := &in.AbortScaleDownDelaySeconds, &out.AbortScaleDownDelaySeconds + *out = new(int32) + **out = **in + } return } diff --git a/rollout/replicaset.go b/rollout/replicaset.go index f7b74059af..39f845ed59 100644 --- a/rollout/replicaset.go +++ b/rollout/replicaset.go @@ -38,6 +38,31 @@ func (c *rolloutContext) removeScaleDownDelay(rs *appsv1.ReplicaSet) error { return err } +// addScaleDownDelay injects the `scale-down-deadline` annotation to the ReplicaSet, or if +// scaleDownDelaySeconds is zero, removes it if it exists +func (c *rolloutContext) addScaleDownDelay2(rs *appsv1.ReplicaSet, scaleDownDelaySeconds time.Duration) error { + if rs == nil { + return nil + } + ctx := context.TODO() + // scaleDownDelaySeconds := time.Duration(defaults.GetScaleDownDelaySecondsOrDefault(c.rollout)) + if scaleDownDelaySeconds == 0 { + // If scaledown deadline is zero, it means we need to remove any replicasets with the delay + // This might happen if we switch from canary with traffic routing to basic canary + if replicasetutil.HasScaleDownDeadline(rs) { + return c.removeScaleDownDelay(rs) + } + return nil + } + deadline := metav1.Now().Add(scaleDownDelaySeconds * time.Second).UTC().Format(time.RFC3339) + patch := fmt.Sprintf(addScaleDownAtAnnotationsPatch, v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey, deadline) + _, err := c.kubeclientset.AppsV1().ReplicaSets(rs.Namespace).Patch(ctx, rs.Name, patchtypes.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + if err == nil { + c.log.Infof("Set '%s' annotation on '%s' to %s (%ds)", v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey, rs.Name, deadline, scaleDownDelaySeconds) + } + return err +} + // addScaleDownDelay injects the `scale-down-deadline` annotation to the ReplicaSet, or if // scaleDownDelaySeconds is zero, removes it if it exists func (c *rolloutContext) addScaleDownDelay(rs *appsv1.ReplicaSet) error { @@ -91,11 +116,14 @@ func (c *Controller) getReplicaSetsForRollouts(r *v1alpha1.Rollout) ([]*appsv1.R return cm.ClaimReplicaSets(rsList) } -// removeScaleDownDelays removes the scale-down-deadline annotation from the new/stable ReplicaSets, +// removeScaleDownDeadlines removes the scale-down-deadline annotation from the new/stable ReplicaSets, // in the event that we moved back to an older revision that is still within its scaleDownDelay. func (c *rolloutContext) removeScaleDownDeadlines() error { var toRemove []*appsv1.ReplicaSet - if c.newRS != nil { + + abortScaleDownDelaySeconds := time.Duration(defaults.GetAbortScaleDownDelaySecondsOrDefault(c.rollout)) + if c.newRS != nil && (c.pauseContext == nil || !c.pauseContext.IsAborted() || abortScaleDownDelaySeconds == 0) { + c.log.Info("hkang removeScaleDownDeadlines") toRemove = append(toRemove, c.newRS) } if c.stableRS != nil { @@ -121,12 +149,47 @@ func (c *rolloutContext) reconcileNewReplicaSet() (bool, error) { return false, err } - if c.pauseContext != nil && c.pauseContext.IsAborted() && c.rollout.Spec.ScaleDownOnAbort { - c.log.Info("scale down on abort") - newReplicasCount = int32(0) + abortScaleDownDelaySeconds := time.Duration(defaults.GetAbortScaleDownDelaySecondsOrDefault(c.rollout)) + if c.pauseContext != nil && c.pauseContext.IsAborted() && abortScaleDownDelaySeconds > 0 { + c.log.Info("scale down on abort for the newRS") + // newReplicasCount = int32(0) + + // if the newRS has scale down annotation and to be scaled down now, set + // Update newReplicasCount = int32(0) + if scaleDownAtStr, ok := c.newRS.Annotations[v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey]; ok { + c.log.Info("scaledown annotation exists: ", scaleDownAtStr) + + c.log.Infof("New rs '%s' has scaledown delay annotation", c.newRS.Name) + scaleDownAtTime, err := time.Parse(time.RFC3339, scaleDownAtStr) + if err != nil { + c.log.Warnf("Unable to read scaleDownAt label on rs '%s'", c.newRS.Name) + } else { + now := metav1.Now() + scaleDownAt := metav1.NewTime(scaleDownAtTime) + if scaleDownAt.After(now.Time) { + c.log.Infof("RS '%s' has not reached the scaleDownTime", c.newRS.Name) + remainingTime := scaleDownAt.Sub(now.Time) + if remainingTime < c.resyncPeriod { + c.enqueueRolloutAfter(c.rollout, remainingTime) + return false, nil + } + } else { + c.log.Infof("RS '%s' has reached the scaleDownTime", c.newRS.Name) + newReplicasCount = int32(0) + } + } + } else { + c.log.Info("Add scale down annotation") + err = c.addScaleDownDelay2(c.newRS, abortScaleDownDelaySeconds) + if err != nil { + return false, err + } + } } + c.log.Info("hkang newReplicasCount:", newReplicasCount) scaled, _, err := c.scaleReplicaSetAndRecordEvent(c.newRS, newReplicasCount) + c.log.Info("hkang after scaleReplicaSetAndRecordEvent, scaled:", scaled) return scaled, err } diff --git a/rollout/replicaset_test.go b/rollout/replicaset_test.go index c0d530951a..475adaadf6 100644 --- a/rollout/replicaset_test.go +++ b/rollout/replicaset_test.go @@ -121,40 +121,55 @@ func TestGetReplicaSetsForRollouts(t *testing.T) { func TestReconcileNewReplicaSet(t *testing.T) { tests := []struct { - name string - rolloutReplicas int - newReplicas int - scaleExpected bool - ScaleDownOnAbort bool - expectedNewReplicas int + name string + rolloutReplicas int + newReplicas int + scaleExpected bool + // ScaleDownOnAbort bool + abortScaleDownDelaySeconds int32 + abortScaleDownTrigger bool + expectedNewReplicas int }{ + /* + { + name: "New Replica Set matches rollout replica: No scale", + rolloutReplicas: 10, + newReplicas: 10, + scaleExpected: false, + }, + { + name: "New Replica Set higher than rollout replica: Scale down", + rolloutReplicas: 10, + newReplicas: 12, + scaleExpected: true, + expectedNewReplicas: 10, + }, + { + name: "New Replica Set lower than rollout replica: Scale up", + rolloutReplicas: 10, + newReplicas: 8, + scaleExpected: true, + expectedNewReplicas: 10, + }, + */ { - name: "New Replica Set matches rollout replica: No scale", + name: "New Replica scaled down to 0: scale down on abort - triggered", rolloutReplicas: 10, newReplicas: 10, - scaleExpected: false, + // ScaleDownOnAbort: true, + abortScaleDownDelaySeconds: 5, + abortScaleDownTrigger: true, + scaleExpected: true, + expectedNewReplicas: 0, }, { - name: "New Replica Set higher than rollout replica: Scale down", - rolloutReplicas: 10, - newReplicas: 12, - scaleExpected: true, - expectedNewReplicas: 10, - }, - { - name: "New Replica Set lower than rollout replica: Scale up", - rolloutReplicas: 10, - newReplicas: 8, - scaleExpected: true, - expectedNewReplicas: 10, - }, - { - name: "New Replica scaled down to 0: scale down on abort", - rolloutReplicas: 10, - newReplicas: 10, - ScaleDownOnAbort: true, - scaleExpected: true, - expectedNewReplicas: 0, + name: "New Replica scaled down to 0: scale down on abort - add annotation", + rolloutReplicas: 10, + newReplicas: 10, + abortScaleDownDelaySeconds: 5, + abortScaleDownTrigger: false, + scaleExpected: false, + expectedNewReplicas: 0, }, } for i := range tests { @@ -175,12 +190,22 @@ func TestReconcileNewReplicaSet(t *testing.T) { recorder: record.NewFakeEventRecorder(), }, } - if test.ScaleDownOnAbort { + if test.abortScaleDownDelaySeconds > 0 { roCtx.pauseContext = &pauseContext{ rollout: rollout, } rollout.Status.Abort = true - rollout.Spec.ScaleDownOnAbort = true + // rollout.Spec.ScaleDownOnAbort = true + rollout.Spec.Strategy = v1alpha1.RolloutStrategy{ + BlueGreen: &v1alpha1.BlueGreenStrategy{ + AbortScaleDownDelaySeconds: &test.abortScaleDownDelaySeconds, + }, + } + + if test.abortScaleDownTrigger { + deadline := metav1.Now().Add(-time.Duration(test.abortScaleDownDelaySeconds) * time.Second).UTC().Format(time.RFC3339) + roCtx.newRS.Annotations[v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey] = deadline + } } scaled, err := roCtx.reconcileNewReplicaSet() diff --git a/rollout/sync.go b/rollout/sync.go index bc5088d07f..16c97cf281 100644 --- a/rollout/sync.go +++ b/rollout/sync.go @@ -390,7 +390,10 @@ func (c *rolloutContext) scaleReplicaSet(rs *appsv1.ReplicaSet, newScale int32, oldScale := defaults.GetReplicasOrDefault(rs.Spec.Replicas) *(rsCopy.Spec.Replicas) = newScale annotations.SetReplicasAnnotations(rsCopy, rolloutReplicas) - if fullScaleDown { + + abortScaleDownDelaySeconds := time.Duration(defaults.GetAbortScaleDownDelaySecondsOrDefault(c.rollout)) + if fullScaleDown && (c.pauseContext == nil || !c.pauseContext.IsAborted() || abortScaleDownDelaySeconds == 0) { + c.log.Info("hkang delete scale down annotation") delete(rsCopy.Annotations, v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey) } rs, err = c.kubeclientset.AppsV1().ReplicaSets(rsCopy.Namespace).Update(ctx, rsCopy, metav1.UpdateOptions{}) @@ -690,6 +693,8 @@ func (c *rolloutContext) persistRolloutStatus(newStatus *v1alpha1.RolloutStatus) ctx := context.TODO() logCtx := logutil.WithVersionFields(c.log, c.rollout) + logCtx.Infof("hkang persistRolloutStatus") + prevStatus := c.rollout.Status c.pauseContext.CalculatePauseStatus(newStatus) newStatus.ObservedGeneration = strconv.Itoa(int(c.rollout.Generation)) diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index a4a0bfa8b5..f3f4008c6a 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -483,6 +483,7 @@ func (s *CanarySuite) TestCanaryScaleDownOnAbort() { WaitForRolloutStatus("Paused"). AbortRollout(). WaitForRolloutStatus("Degraded"). + Sleep(3*time.Second). Then(). ExpectRevisionPodCount("2", 0) } diff --git a/test/e2e/functional/canary-scaledownonabort.yaml b/test/e2e/functional/canary-scaledownonabort.yaml index 6a3fc6cdfb..8d5d05aff3 100644 --- a/test/e2e/functional/canary-scaledownonabort.yaml +++ b/test/e2e/functional/canary-scaledownonabort.yaml @@ -79,9 +79,9 @@ spec: requests: memory: 16Mi cpu: 5m - scaleDownOnAbort: true strategy: canary: + abortScaleDownDelaySeconds: 1 scaleDownDelayRevisionLimit: 1 canaryService: canary-scaledowndelay-canary stableService: canary-scaledowndelay-stable diff --git a/test/e2e/functional_test.go b/test/e2e/functional_test.go index fcd10d9a05..203b6b2be1 100644 --- a/test/e2e/functional_test.go +++ b/test/e2e/functional_test.go @@ -948,9 +948,9 @@ metadata: name: bluegreen-scaledown-on-abort spec: replicas: 2 - scaleDownOnAbort: true strategy: blueGreen: + abortScaleDownDelaySeconds: 1 activeService: bluegreen-preview-replicas-active previewService: bluegreen-preview-replicas-preview previewReplicaCount: 1 @@ -983,6 +983,7 @@ spec: When(). AbortRollout(). WaitForRolloutStatus("Degraded"). + Sleep(3*time.Second). Then(). ExpectRevisionPodCount("2", 0) } diff --git a/utils/annotations/annotations.go b/utils/annotations/annotations.go index c2da999882..4cfe769e2b 100644 --- a/utils/annotations/annotations.go +++ b/utils/annotations/annotations.go @@ -84,8 +84,10 @@ func ReplicasAnnotationsNeedUpdate(rs *appsv1.ReplicaSet, desiredReplicas int32) } hasScaleDownDelay := rs.Annotations[v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey] if desiredReplicas == int32(0) && hasScaleDownDelay != "" { + fmt.Println("hkang desiredReplicas is zero and hasScaleDownDelay", hasScaleDownDelay) return true } + fmt.Println("hkang desiredReplicas is NOT zero or hasScaleDownDelay", hasScaleDownDelay) return false } diff --git a/utils/defaults/defaults.go b/utils/defaults/defaults.go index 6707086794..cafeb217ad 100644 --- a/utils/defaults/defaults.go +++ b/utils/defaults/defaults.go @@ -23,6 +23,8 @@ const ( DefaultProgressDeadlineSeconds = int32(600) // DefaultScaleDownDelaySeconds default seconds before scaling down old replicaset after switching services DefaultScaleDownDelaySeconds = int32(30) + // DefaultAbortScaleDownDelaySeconds default seconds before scaling down old replicaset after switching services + DefaultAbortScaleDownDelaySeconds = int32(0) // DefaultAutoPromotionEnabled default value for auto promoting a blueGreen strategy DefaultAutoPromotionEnabled = true // DefaultConsecutiveErrorLimit is the default number times a metric can error in sequence before @@ -110,6 +112,24 @@ func GetScaleDownDelaySecondsOrDefault(rollout *v1alpha1.Rollout) int32 { return 0 } +func GetAbortScaleDownDelaySecondsOrDefault(rollout *v1alpha1.Rollout) int32 { + if rollout.Spec.Strategy.BlueGreen != nil { + if rollout.Spec.Strategy.BlueGreen.AbortScaleDownDelaySeconds != nil { + return *rollout.Spec.Strategy.BlueGreen.AbortScaleDownDelaySeconds + } + return DefaultAbortScaleDownDelaySeconds + } + if rollout.Spec.Strategy.Canary != nil { + if rollout.Spec.Strategy.Canary.TrafficRouting != nil { + if rollout.Spec.Strategy.Canary.AbortScaleDownDelaySeconds != nil { + return *rollout.Spec.Strategy.Canary.AbortScaleDownDelaySeconds + } + return DefaultAbortScaleDownDelaySeconds + } + } + return 0 +} + func GetAutoPromotionEnabledOrDefault(rollout *v1alpha1.Rollout) bool { if rollout.Spec.Strategy.BlueGreen == nil { return DefaultAutoPromotionEnabled diff --git a/utils/replicaset/canary.go b/utils/replicaset/canary.go index 3ba8670048..701c842a26 100644 --- a/utils/replicaset/canary.go +++ b/utils/replicaset/canary.go @@ -297,10 +297,6 @@ func GetCanaryReplicasOrWeight(rollout *v1alpha1.Rollout) (*int32, int32) { return nil, 100 } if scs := UseSetCanaryScale(rollout); scs != nil { - if rollout.Status.Abort && rollout.Spec.ScaleDownOnAbort { - return nil, 0 - } - if scs.Replicas != nil { return scs.Replicas, 0 } else if scs.Weight != nil { diff --git a/utils/replicaset/canary_test.go b/utils/replicaset/canary_test.go index 405245a70c..ac4a548865 100644 --- a/utils/replicaset/canary_test.go +++ b/utils/replicaset/canary_test.go @@ -80,7 +80,6 @@ func TestCalculateReplicaCountsForCanary(t *testing.T) { maxSurge intstr.IntOrString maxUnavailable intstr.IntOrString promoteFull bool - scaleDownOnAbort bool stableSpecReplica int32 stableAvailableReplica int32 @@ -572,29 +571,12 @@ func TestCalculateReplicaCountsForCanary(t *testing.T) { expectedStableReplicaCount: 4, expectedCanaryReplicaCount: 3, }, - { - name: "Ignore setCanaryScale when scaleDownonAbort is true and rollout is aborted", - rolloutSpecReplicas: 10, - setWeight: 20, - scaleDownOnAbort: true, - - stableSpecReplica: 10, - stableAvailableReplica: 10, - - setCanaryScale: newSetCanaryScale(intPnt(3), intPnt(25), false), - trafficRouting: &v1alpha1.RolloutTrafficRouting{}, - - expectedStableReplicaCount: 10, - expectedCanaryReplicaCount: 0, - }, } for i := range tests { test := tests[i] t.Run(test.name, func(t *testing.T) { rollout := newRollout(test.rolloutSpecReplicas, test.setWeight, test.maxSurge, test.maxUnavailable, "canary", "stable", test.setCanaryScale, test.trafficRouting) rollout.Status.PromoteFull = test.promoteFull - rollout.Spec.ScaleDownOnAbort = test.scaleDownOnAbort - rollout.Status.Abort = test.scaleDownOnAbort stableRS := newRS("stable", test.stableSpecReplica, test.stableAvailableReplica) canaryRS := newRS("canary", test.canarySpecReplica, test.canaryAvailableReplica) newRSReplicaCount, stableRSReplicaCount := CalculateReplicaCountsForCanary(rollout, canaryRS, stableRS, []*appsv1.ReplicaSet{test.olderRS})