move cloud discovery after nodeInterval / drainInterval wait (#270)

Signed-off-by: sbadiger <[email protected]>
keikoproj · Jun 29, 2021 · c0a163b · c0a163b
1 parent b15838e
commit c0a163b
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 17 deletions.
diff --git a/controllers/rollingupgrade_controller.go b/controllers/rollingupgrade_controller.go
@@ -146,13 +146,6 @@ func (r *RollingUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		RollingUpgrade: rollingUpgrade,
 		metricsMutex:   &sync.Mutex{},
 	}
-	rollupCtx.Cloud = NewDiscoveredState(rollupCtx.Auth, rollupCtx.Logger)
-	if err := rollupCtx.Cloud.Discover(); err != nil {
-		r.Info("failed to discover the cloud", "scalingGroup", scalingGroupName, "name", rollingUpgrade.NamespacedName())
-		rollingUpgrade.SetCurrentStatus(v1alpha1.StatusError)
-		common.SetMetricRollupFailed(rollingUpgrade.Name)
-		return ctrl.Result{}, err
-	}
 
 	// process node rotation
 	if err := rollupCtx.RotateNodes(); err != nil {

diff --git a/controllers/upgrade.go b/controllers/upgrade.go
@@ -59,24 +59,15 @@ type RollingUpgradeContext struct {
 	metricsMutex   *sync.Mutex
 }
 
-// TODO: main node rotation logic
 func (r *RollingUpgradeContext) RotateNodes() error {
 	var (
 		lastTerminationTime = r.RollingUpgrade.LastNodeTerminationTime()
 		nodeInterval        = r.RollingUpgrade.NodeIntervalSeconds()
 		lastDrainTime       = r.RollingUpgrade.LastNodeDrainTime()
 		drainInterval       = r.RollingUpgrade.PostDrainDelaySeconds()
 	)
-	r.RollingUpgrade.SetCurrentStatus(v1alpha1.StatusRunning)
-	common.SetMetricRollupInitOrRunning(r.RollingUpgrade.Name)
-
-	// set status start time
-	if r.RollingUpgrade.StartTime() == "" {
-		r.RollingUpgrade.SetStartTime(time.Now().Format(time.RFC3339))
-	}
 
 	if !lastTerminationTime.IsZero() || !lastDrainTime.IsZero() {
-
 		// Check if we are still waiting on a termination delay
 		if time.Since(lastTerminationTime.Time).Seconds() < float64(nodeInterval) {
 			r.Info("reconcile requeue due to termination interval wait", "name", r.RollingUpgrade.NamespacedName())
@@ -90,6 +81,22 @@ func (r *RollingUpgradeContext) RotateNodes() error {
 		}
 	}
 
+	// set status start time
+	if r.RollingUpgrade.StartTime() == "" {
+		r.RollingUpgrade.SetStartTime(time.Now().Format(time.RFC3339))
+	}
+	r.RollingUpgrade.SetCurrentStatus(v1alpha1.StatusRunning)
+	common.SetMetricRollupInitOrRunning(r.RollingUpgrade.Name)
+
+	// discover the state of AWS and K8s cluster.
+	r.Cloud = NewDiscoveredState(r.Auth, r.Logger)
+	if err := r.Cloud.Discover(); err != nil {
+		r.Info("failed to discover the cloud", "scalingGroup", r.RollingUpgrade.ScalingGroupName(), "name", r.RollingUpgrade.NamespacedName())
+		r.RollingUpgrade.SetCurrentStatus(v1alpha1.StatusError)
+		common.SetMetricRollupFailed(r.RollingUpgrade.Name)
+		return err
+	}
+
 	var (
 		scalingGroup = awsprovider.SelectScalingGroup(r.RollingUpgrade.ScalingGroupName(), r.Cloud.ScalingGroups)
 	)
@@ -245,6 +252,7 @@ func (r *RollingUpgradeContext) ReplaceNodeBatch(batch []*autoscaling.Instance)
 					if err := r.Auth.DrainNode(&node, time.Duration(r.RollingUpgrade.PostDrainDelaySeconds()), r.RollingUpgrade.DrainTimeout(), r.Auth.Kubernetes); err != nil {
 						if !r.RollingUpgrade.IsIgnoreDrainFailures() {
 							r.DrainManager.DrainErrors <- errors.Errorf("DrainNode failed: instanceID - %v, %v", instanceID, err.Error())
+							//TODO: BREAK AFTER ERRORS?
 						}
 					}
 				}
@@ -326,7 +334,6 @@ func (r *RollingUpgradeContext) ReplaceNodeBatch(batch []*autoscaling.Instance)
 			r.NodeStep(inProcessingNodes, nodeSteps, r.RollingUpgrade.Spec.AsgName, nodeName, v1alpha1.NodeRotationTerminated)
 			r.DoNodeStep(inProcessingNodes, nodeSteps, r.RollingUpgrade.Spec.AsgName, nodeName, v1alpha1.NodeRotationCompleted, terminatedTime)
 		}
-
 		r.UpdateMetricsStatus(inProcessingNodes, nodeSteps)
 
 	case <-time.After(DefaultWaitGroupTimeout):