From c2fdb372ad4e70f2548512234d9ca22f8e8673f3 Mon Sep 17 00:00:00 2001 From: Shreyas Badiger <7680410+shreyas-badiger@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:51:27 -0800 Subject: [PATCH] Early cordon (#405) * early-cordon nodes Signed-off-by: sbadiger * early cordon Signed-off-by: sbadiger * include context in cordon and drain functions * cordon only drifted instances * add unit tests * Update aws-sdk-go-cache to v0.0.2 (#399) Signed-off-by: Todd Ekenstam Signed-off-by: sbadiger * Process drain-failure nodes at the end (#394) * Process drain-failures at the end Signed-off-by: ssheladiya Signed-off-by: sbadiger * early-cordon nodes Signed-off-by: sbadiger * early cordon Signed-off-by: sbadiger * include context in cordon and drain functions Signed-off-by: sbadiger * Release v1.0.8 (#400) Signed-off-by: sbadiger * cordon only drifted instances Signed-off-by: sbadiger * add unit tests Signed-off-by: sbadiger * resolve merge conflicts * update go.sum * resolve test errors * remove cordon as upgrade strategy * remove space * improve test coverage * improve code coverage * remove redundant code * remove unused imports * Update controllers/providers/kubernetes/nodes.go Co-authored-by: Venkata Gunapati * uncordon the nodes * error handling for uncordoning * add tests * handle uncordon scenario properly * Update controllers/providers/kubernetes/nodes.go Co-authored-by: Venkata Gunapati * fix typo * fix lint errors * default the feature to false --------- Signed-off-by: sbadiger Signed-off-by: Todd Ekenstam Co-authored-by: Todd Ekenstam <3845995+tekenstam@users.noreply.github.com> Co-authored-by: Siddharth Sheladiya <130721367+ssheladiya@users.noreply.github.com> Co-authored-by: Venkata Gunapati --- Makefile | 1 + controllers/cloud.go | 7 +- controllers/helpers_test.go | 4 + controllers/metrics_test.go | 4 +- controllers/providers/aws/ec2.go | 39 ++ controllers/providers/kubernetes/nodes.go | 31 ++ controllers/rollingupgrade_controller.go | 8 + controllers/upgrade.go | 63 +++ controllers/upgrade_test.go | 63 +++ coverage.txt | 575 ++++++++++++---------- main.go | 3 + 11 files changed, 544 insertions(+), 254 deletions(-) diff --git a/Makefile b/Makefile index f8155010..228f9a4c 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ all: manager # Run tests ENVTEST_ASSETS_DIR=$(shell pwd)/testbin test: manifests generate fmt vet envtest + go test ./controllers/... ./api/... go tool cover -html=./coverage.txt -o cover.html # Build manager binary diff --git a/controllers/cloud.go b/controllers/cloud.go index 1c2bb4ce..bcce7c1f 100644 --- a/controllers/cloud.go +++ b/controllers/cloud.go @@ -27,9 +27,10 @@ import ( ) var ( - instanceStateTagKey = "upgrademgr.keikoproj.io/state" - inProgressTagValue = "in-progress" - failedDrainTagValue = "failed-drain" + instanceStateTagKey = "upgrademgr.keikoproj.io/state" + inProgressTagValue = "in-progress" + failedDrainTagValue = "failed-drain" + earlyCordonedTagValue = "early-cordoned" ) type DiscoveredState struct { diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go index 3c730e7a..54fd6724 100644 --- a/controllers/helpers_test.go +++ b/controllers/helpers_test.go @@ -426,3 +426,7 @@ func (mockAutoscalingGroup MockAutoscalingGroup) EnterStandby(_ *autoscaling.Ent output := &autoscaling.EnterStandbyOutput{} return output, nil } + +func (m *MockEC2) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { + return &ec2.DeleteTagsOutput{}, nil +} diff --git a/controllers/metrics_test.go b/controllers/metrics_test.go index a55d7025..7ba2d122 100644 --- a/controllers/metrics_test.go +++ b/controllers/metrics_test.go @@ -11,8 +11,8 @@ import ( func TestNodeTurnsOntoStep(t *testing.T) { g := gomega.NewGomegaWithT(t) - reconsiler := createRollingUpgradeReconciler(t) - r := createRollingUpgradeContext(reconsiler) + reconciler := createRollingUpgradeReconciler(t) + r := createRollingUpgradeContext(reconciler) //A map to retain the steps for multiple nodes nodeSteps := make(map[string][]v1alpha1.NodeStepDuration) diff --git a/controllers/providers/aws/ec2.go b/controllers/providers/aws/ec2.go index 51cc0dd1..36ae1a21 100644 --- a/controllers/providers/aws/ec2.go +++ b/controllers/providers/aws/ec2.go @@ -62,6 +62,31 @@ func (a *AmazonClientSet) DescribeTaggedInstanceIDs(tagKey, tagValue string) ([] return instances, err } +func (a *AmazonClientSet) DescribeInstancesWithoutTagValue(tagKey string, tagValue string) ([]string, error) { + instances := []string{} + input := &ec2.DescribeInstancesInput{} + tagAndValueIsPresent := false + + err := a.Ec2Client.DescribeInstancesPages(input, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { + for _, res := range page.Reservations { + for _, instance := range res.Instances { + for _, t := range instance.Tags { + if *t.Key == tagKey && *t.Value == tagValue { + tagAndValueIsPresent = true + break + } + } + if !tagAndValueIsPresent { + instances = append(instances, aws.StringValue(instance.InstanceId)) + } + tagAndValueIsPresent = false + } + } + return page.NextToken != nil + }) + return instances, err +} + func (a *AmazonClientSet) TagEC2instances(instanceIDs []string, tagKey, tagValue string) error { input := &ec2.CreateTagsInput{ Resources: aws.StringSlice(instanceIDs), @@ -75,3 +100,17 @@ func (a *AmazonClientSet) TagEC2instances(instanceIDs []string, tagKey, tagValue _, err := a.Ec2Client.CreateTags(input) return err } + +func (a *AmazonClientSet) UntagEC2instances(instanceIDs []string, tagKey, tagValue string) error { + input := &ec2.DeleteTagsInput{ + Resources: aws.StringSlice(instanceIDs), + Tags: []*ec2.Tag{ + { + Key: aws.String(tagKey), + Value: aws.String(tagValue), + }, + }, + } + _, err := a.Ec2Client.DeleteTags(input) + return err +} diff --git a/controllers/providers/kubernetes/nodes.go b/controllers/providers/kubernetes/nodes.go index 7a24ee5b..e0d43d55 100644 --- a/controllers/providers/kubernetes/nodes.go +++ b/controllers/providers/kubernetes/nodes.go @@ -50,6 +50,7 @@ func (k *KubernetesClientSet) DrainNode(node *corev1.Node, PostDrainDelaySeconds } helper := &drain.Helper{ + Ctx: context.Background(), Client: client, Force: true, GracePeriodSeconds: -1, @@ -75,3 +76,33 @@ func (k *KubernetesClientSet) DrainNode(node *corev1.Node, PostDrainDelaySeconds } return nil } + +// CordonUncordonNode cordons a node. +func (k *KubernetesClientSet) CordonUncordonNode(node *corev1.Node, client kubernetes.Interface, cordonNode bool) error { + if client == nil { + return fmt.Errorf("K8sClient not set") + } + + if node == nil { + return fmt.Errorf("node not set") + } + + helper := &drain.Helper{ + Ctx: context.Background(), + Client: client, + Force: true, + GracePeriodSeconds: -1, + IgnoreAllDaemonSets: true, + Out: os.Stdout, + ErrOut: os.Stdout, + DeleteEmptyDirData: true, + } + + if err := drain.RunCordonOrUncordon(helper, node, cordonNode); err != nil { + if apierrors.IsNotFound(err) { + return err + } + return fmt.Errorf("error cordoning node: %v", err) + } + return nil +} diff --git a/controllers/rollingupgrade_controller.go b/controllers/rollingupgrade_controller.go index 6c09fae3..f57773db 100644 --- a/controllers/rollingupgrade_controller.go +++ b/controllers/rollingupgrade_controller.go @@ -58,6 +58,7 @@ type RollingUpgradeReconciler struct { IgnoreDrainFailures bool ReplacementNodesMap *sync.Map MaxReplacementNodes int + EarlyCordonNodes bool } // RollingUpgradeAuthenticator has the clients for providers @@ -212,6 +213,7 @@ func (r *RollingUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Reque IgnoreDrainFailures: r.IgnoreDrainFailures, ReplacementNodesMap: r.ReplacementNodesMap, MaxReplacementNodes: r.MaxReplacementNodes, + EarlyCordonNodes: r.EarlyCordonNodes, } // process node rotation @@ -219,6 +221,12 @@ func (r *RollingUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Reque rollingUpgrade.SetCurrentStatus(v1alpha1.StatusError) rollingUpgrade.SetLabel(v1alpha1.LabelKeyRollingUpgradeCurrentStatus, v1alpha1.StatusError) common.SetMetricRollupFailed(rollingUpgrade.Name) + + // try to uncordon all the cordoned nodes. + if _, err2 := rollupCtx.CordonUncordonAllNodes(false); err2 != nil { + r.Error(err2, "failed to uncordon the nodes.", "name", rollingUpgrade.NamespacedName()) + } + return ctrl.Result{}, err } diff --git a/controllers/upgrade.go b/controllers/upgrade.go index 4047a0fd..a36dc6c4 100644 --- a/controllers/upgrade.go +++ b/controllers/upgrade.go @@ -65,6 +65,7 @@ type RollingUpgradeContext struct { ReplacementNodesMap *sync.Map MaxReplacementNodes int AllowReplacements bool + EarlyCordonNodes bool } func (r *RollingUpgradeContext) RotateNodes() error { @@ -147,6 +148,13 @@ func (r *RollingUpgradeContext) ReplaceNodeBatch(batch []*autoscaling.Instance) inProcessingNodes = make(map[string]*v1alpha1.NodeInProcessing) } + //Early-Cordon - Cordon all the nodes to avoid any further scheduling of new pods. + if r.EarlyCordonNodes { + if ok, err := r.CordonUncordonAllNodes(true); !ok { + return ok, err + } + } + switch mode { case v1alpha1.UpdateStrategyModeEager: for _, target := range batch { @@ -750,3 +758,58 @@ func (r *RollingUpgradeContext) ClusterBallooning(batchSize int) (bool, int) { } return false, batchSize } + +func (r *RollingUpgradeContext) CordonUncordonAllNodes(cordonNode bool) (bool, error) { + scalingGroup := awsprovider.SelectScalingGroup(r.RollingUpgrade.ScalingGroupName(), r.Cloud.ScalingGroups) + var instanceIDs []string + var err error + + if cordonNode { + instanceIDs, err = r.Cloud.AmazonClientSet.DescribeInstancesWithoutTagValue(instanceStateTagKey, earlyCordonedTagValue) + if err != nil { + r.Error(err, "failed to describe instances for early-cordoning", "name", r.RollingUpgrade.NamespacedName()) + return false, errors.Wrap(err, "failed to describe instances for early-cordoning") + } + } else { + instanceIDs, err = r.Auth.DescribeTaggedInstanceIDs(instanceStateTagKey, earlyCordonedTagValue) + if err != nil { + r.Error(err, "failed to discover ec2 instances with early-cordoned tag", "name", r.RollingUpgrade.NamespacedName()) + } + + r.Info("removing early-cordoning tag while uncordoning instances", "name", r.RollingUpgrade.NamespacedName()) + if err := r.Auth.UntagEC2instances(instanceIDs, instanceStateTagKey, earlyCordonedTagValue); err != nil { + r.Error(err, "failed to delete early-cordoned tag for instances", "name", r.RollingUpgrade.NamespacedName()) + } + // add unit test as well. + + } + + for _, instanceID := range instanceIDs { + if instance := awsprovider.SelectScalingGroupInstance(instanceID, scalingGroup); !reflect.DeepEqual(instance, &autoscaling.Instance{}) { + //Don't consider if the instance is in terminating state. + if !common.ContainsEqualFold(awsprovider.TerminatingInstanceStates, aws.StringValue(instance.LifecycleState)) { + node := kubeprovider.SelectNodeByInstanceID(*instance.InstanceId, r.Cloud.ClusterNodes) + if node == nil { + r.Info("node object not found in clusterNodes, unable to early-cordon node", "instanceID", instance.InstanceId, "name", r.RollingUpgrade.NamespacedName()) + continue + } + //Early cordon only the dirfted instances and not the instances that have same scaling-config as the scaling-group + if !r.IsInstanceDrifted(instance) { + break + } + r.Info("early cordoning node", "instanceID", instance.InstanceId, "name", r.RollingUpgrade.NamespacedName()) + if err := r.Auth.CordonUncordonNode(node, r.Auth.Kubernetes, cordonNode); err != nil { + r.Error(err, "failed to early cordon the nodes", "instanceID", instance.InstanceId, "name", r.RollingUpgrade.NamespacedName()) + return false, err + } + // Set instance-state to early-cordoned tag + r.Info("tagging instances with cordoned=true", "instanceID", instance.InstanceId, "name", r.RollingUpgrade.NamespacedName()) + if err := r.Auth.TagEC2instances([]string{*instance.InstanceId}, instanceStateTagKey, earlyCordonedTagValue); err != nil { + r.Error(err, "failed to tag instances with cordoned=true", "instanceID", instance.InstanceId, "name", r.RollingUpgrade.NamespacedName()) + return true, err + } + } + } + } + return true, nil +} diff --git a/controllers/upgrade_test.go b/controllers/upgrade_test.go index ca2ff69e..a2a5e4ea 100644 --- a/controllers/upgrade_test.go +++ b/controllers/upgrade_test.go @@ -1,6 +1,7 @@ package controllers import ( + "context" "os" "testing" @@ -105,6 +106,7 @@ func TestRunCordonOrUncordon(t *testing.T) { for _, test := range tests { rollupCtx := createRollingUpgradeContext(test.Reconciler) helper := &drain.Helper{ + Ctx: context.Background(), Client: rollupCtx.Auth.Kubernetes, Force: true, GracePeriodSeconds: -1, @@ -299,6 +301,7 @@ func TestRotateNodes(t *testing.T) { rollupCtx := test.RollingUpgradeContext rollupCtx.Cloud.ScalingGroups = test.AsgClient.autoScalingGroups rollupCtx.Auth.AmazonClientSet.AsgClient = test.AsgClient + rollupCtx.EarlyCordonNodes = true err := rollupCtx.RotateNodes() if err != nil { @@ -507,6 +510,7 @@ func TestIgnoreDrainFailuresAndDrainTimeout(t *testing.T) { rollupCtx.Cloud.ScalingGroups = test.AsgClient.autoScalingGroups rollupCtx.Cloud.ClusterNodes = test.ClusterNodes rollupCtx.Auth.AmazonClientSet.AsgClient = test.AsgClient + rollupCtx.EarlyCordonNodes = true err := rollupCtx.RotateNodes() if err != nil { @@ -545,6 +549,7 @@ func TestClusterBallooning(t *testing.T) { reconciler := createRollingUpgradeReconciler(t) reconciler.MaxReplacementNodes = 500 reconciler.ReplacementNodesMap.Store("ReplacementNodes", 500) + reconciler.EarlyCordonNodes = true return reconciler }(), createRollingUpgrade(), @@ -617,3 +622,61 @@ func TestClusterBallooning(t *testing.T) { } } + +func TestCordoningAndUncordoningOfNodes(t *testing.T) { + var tests = []struct { + TestDescription string + Reconciler *RollingUpgradeReconciler + Node *corev1.Node + CordonNodeFlag bool + ExpectedUnschdeulableValue bool + ExpectedError bool + }{ + { + "Test if all the nodes are cordoned.", + createRollingUpgradeReconciler(t), + createNode("mock-node-1"), + true, + true, + false, + }, + { + "Test if all the nodes are uncordoned", + createRollingUpgradeReconciler(t), + createNode("mock-node-1"), + false, + false, + false, + }, + { + "Try to cordon an unknown node.", + createRollingUpgradeReconciler(t), + createNode("mock-node-4"), + true, + true, + true, + }, + } + for _, test := range tests { + rollupCtx := createRollingUpgradeContext(test.Reconciler) + + if err := rollupCtx.Auth.CordonUncordonNode(test.Node, rollupCtx.Auth.Kubernetes, test.CordonNodeFlag); err != nil && test.ExpectedError { + continue + } + + // By default, nodes are uncordoned. Therefore, before testing uncordoning the node, first cordon it. + if !test.CordonNodeFlag { + if err := rollupCtx.Auth.CordonUncordonNode(test.Node, rollupCtx.Auth.Kubernetes, true); err != nil { + t.Errorf("Test Description: %s \n error: %v", test.TestDescription, err) + } + } + + if err := rollupCtx.Auth.CordonUncordonNode(test.Node, rollupCtx.Auth.Kubernetes, test.CordonNodeFlag); err != nil { + t.Errorf("Test Description: %s \n error: %v", test.TestDescription, err) + } + + if test.ExpectedUnschdeulableValue != test.Node.Spec.Unschedulable { + t.Errorf("Test Description: %s \n expectedValue: %v, actualValue: %v", test.TestDescription, test.ExpectedUnschdeulableValue, test.Node.Spec.Unschedulable) + } + } +} diff --git a/coverage.txt b/coverage.txt index 81cf37dc..8426a727 100644 --- a/coverage.txt +++ b/coverage.txt @@ -1,315 +1,392 @@ mode: set github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:59.20,62.2 2 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:65.68,67.50 2 0 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:89.2,89.37 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:67.50,76.17 3 0 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:83.3,85.24 3 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:76.17,77.75 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:77.75,79.5 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:79.10,81.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:83.3,85.24 3 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:86.8,88.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:89.2,89.37 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:92.115,94.39 2 0 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:116.2,116.27 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:94.39,103.17 3 0 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:110.3,112.24 3 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:103.17,104.75 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:104.75,106.5 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:106.10,108.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:110.3,112.24 3 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:113.8,115.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:116.2,116.27 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:119.55,121.2 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:123.68,125.2 1 0 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:128.81,129.42 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:129.42,131.3 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:131.8,133.45 2 1 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:142.3,143.36 2 1 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:165.3,165.38 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:133.45,138.4 4 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:138.9,140.4 1 1 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:142.3,143.36 2 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:143.36,152.18 3 1 -github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:159.4,161.25 3 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:152.18,153.76 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:153.76,155.6 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:155.11,157.6 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:159.4,161.25 3 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:162.9,164.4 1 1 +github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:165.3,165.38 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:169.50,171.2 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:173.46,175.2 1 1 github.com/keikoproj/upgrade-manager/controllers/common/metrics.go:177.43,179.2 1 1 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:24.55,25.29 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:30.2,30.14 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:25.29,26.33 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:26.33,28.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:33.27,34.11 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:37.2,37.10 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:34.11,36.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:40.27,41.11 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:44.2,44.10 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:41.11,43.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:47.58,49.45 2 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:53.2,53.15 1 0 -github.com/keikoproj/upgrade-manager/controllers/common/utils.go:49.45,52.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:43.97,48.2 1 1 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:50.44,53.16 2 1 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:56.2,59.16 3 1 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:62.2,65.16 3 1 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:68.2,70.12 2 1 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:53.16,55.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:59.16,61.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/cloud.go:65.16,67.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:25.55,26.29 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:26.29,27.33 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:27.33,29.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:31.2,31.14 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:34.27,35.11 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:35.11,37.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:38.2,38.10 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:41.27,42.11 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:42.11,44.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:45.2,45.10 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:48.58,50.45 2 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:50.45,53.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:54.2,54.15 1 0 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:58.37,60.38 2 1 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:60.38,63.3 2 1 +github.com/keikoproj/upgrade-manager/controllers/common/utils.go:64.2,64.14 1 1 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:45.97,50.2 1 1 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:52.44,55.16 2 1 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:55.16,57.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:58.2,61.16 3 1 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:61.16,63.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:64.2,67.16 3 1 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:67.16,69.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/cloud.go:70.2,72.12 2 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:12.153,15.2 2 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:18.104,21.28 3 1 -github.com/keikoproj/upgrade-manager/controllers/metrics.go:24.2,24.47 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:21.28,23.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/metrics.go:24.2,24.47 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:28.100,29.30 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:29.30,30.26 1 0 github.com/keikoproj/upgrade-manager/controllers/metrics.go:30.26,32.4 1 0 github.com/keikoproj/upgrade-manager/controllers/metrics.go:37.84,39.55 1 0 -github.com/keikoproj/upgrade-manager/controllers/metrics.go:48.2,54.4 1 0 github.com/keikoproj/upgrade-manager/controllers/metrics.go:39.55,40.33 1 0 github.com/keikoproj/upgrade-manager/controllers/metrics.go:40.33,46.4 3 0 +github.com/keikoproj/upgrade-manager/controllers/metrics.go:48.2,54.4 1 0 github.com/keikoproj/upgrade-manager/controllers/metrics.go:58.164,69.2 2 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:72.139,75.47 2 1 -github.com/keikoproj/upgrade-manager/controllers/metrics.go:87.2,89.48 3 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:75.47,83.3 2 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:83.8,85.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/metrics.go:87.2,89.48 3 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:89.48,99.3 8 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:99.8,99.50 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:99.50,102.26 3 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:102.26,107.4 4 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:113.118,115.2 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:117.147,119.46 2 1 -github.com/keikoproj/upgrade-manager/controllers/metrics.go:127.2,127.25 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:119.46,123.3 1 1 github.com/keikoproj/upgrade-manager/controllers/metrics.go:123.8,126.3 2 1 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:75.106,79.16 4 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:89.2,89.48 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:96.2,97.68 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:103.2,103.47 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:108.2,116.106 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:122.2,122.51 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:133.2,133.16 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:139.2,139.107 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:145.2,164.34 5 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:172.2,172.48 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:178.2,178.73 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:79.16,80.30 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:85.3,85.28 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:80.30,84.4 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:89.48,93.3 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:97.68,101.3 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:103.47,105.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:116.106,119.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:122.51,125.112 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:130.3,130.14 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:125.112,129.4 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:133.16,136.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:139.107,142.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:142.8,144.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:164.34,168.4 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:172.48,176.3 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:182.77,189.2 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:192.76,194.46 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:194.46,196.10 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:202.4,202.15 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:196.10,201.5 4 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:204.46,206.10 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:212.4,212.15 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:206.10,211.5 4 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:214.46,216.10 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:222.4,222.15 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:216.10,221.5 4 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:228.58,229.12 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:229.12,232.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:236.90,238.80 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:238.80,240.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:244.69,248.60 3 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:252.2,252.26 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:255.2,255.21 1 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:248.60,251.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:252.26,254.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/metrics.go:127.2,127.25 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:81.106,85.16 4 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:85.16,86.30 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:86.30,90.4 3 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:91.3,91.28 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:95.2,95.48 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:95.48,99.3 3 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:102.2,103.68 2 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:103.68,107.3 3 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:110.2,110.47 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:110.47,112.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:115.2,118.25 2 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:118.25,122.3 3 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:124.2,130.106 2 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:130.106,133.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:136.2,136.48 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:136.48,140.17 4 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:140.17,143.4 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:144.3,144.47 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:144.47,145.52 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:145.52,147.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:152.2,152.51 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:152.51,155.112 3 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:155.112,159.4 3 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:160.3,160.14 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:163.2,163.16 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:163.16,166.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:169.2,170.33 2 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:170.33,171.83 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:171.83,174.4 2 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:174.9,176.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:177.8,179.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:182.2,182.107 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:182.107,186.3 3 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:186.8,188.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:191.2,206.34 3 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:206.34,210.4 3 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:220.2,220.48 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:220.48,226.70 4 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:226.70,228.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:230.3,230.28 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:233.2,233.73 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:237.77,244.2 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:247.76,249.46 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:249.46,251.10 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:251.10,256.5 4 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:257.4,257.15 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:259.46,261.10 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:261.10,266.5 4 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:267.4,267.15 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:269.46,271.10 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:271.10,276.5 4 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:277.4,277.15 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:283.58,284.12 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:284.12,287.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:291.84,294.80 2 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:294.80,297.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:299.2,299.254 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:299.254,301.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:305.66,306.12 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:306.12,309.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:313.69,317.60 3 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:317.60,320.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:321.2,321.26 1 1 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:321.26,323.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/rollingupgrade_controller.go:324.2,324.21 1 1 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:39.55,43.2 1 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:45.61,56.2 2 1 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:58.86,64.16 5 1 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:67.2,67.25 1 1 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:64.16,66.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:67.2,67.25 1 1 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:70.65,72.18 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:76.2,77.16 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:82.2,84.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:72.18,74.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:76.2,77.16 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:77.16,80.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:82.2,84.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:87.60,89.18 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:93.2,94.16 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:99.2,100.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:89.18,91.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:93.2,94.16 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:94.16,97.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:99.2,100.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:103.61,105.18 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:109.2,110.16 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:115.2,116.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:105.18,107.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:109.2,110.16 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:110.16,113.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:115.2,116.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:119.60,121.18 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:125.2,126.16 2 0 -github.com/keikoproj/upgrade-manager/controllers/script_runner.go:131.2,132.12 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:121.18,123.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:125.2,126.16 2 0 github.com/keikoproj/upgrade-manager/controllers/script_runner.go:126.16,129.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:65.53,71.40 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:76.2,76.43 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:83.2,86.59 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:89.2,100.32 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:107.2,108.57 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:112.2,112.12 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:71.40,73.3 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:76.43,81.3 4 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:86.59,88.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:100.32,105.3 4 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:108.57,110.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:115.95,126.30 5 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:130.2,130.14 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:216.2,224.139 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:229.2,229.122 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:234.2,234.69 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:298.2,299.12 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:304.2,304.9 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:372.2,372.18 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:126.30,128.3 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:131.40,132.32 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:147.3,149.36 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:171.3,171.32 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:185.3,186.29 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:190.3,190.79 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:192.39,193.32 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:207.3,209.111 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:132.32,135.19 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:140.4,144.115 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:135.19,137.13 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:149.36,152.112 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:158.4,159.62 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:164.4,165.20 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:152.112,156.5 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:159.62,161.5 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:166.9,168.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:171.32,174.19 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:178.4,181.124 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:174.19,176.13 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:186.29,189.4 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:193.32,196.19 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:200.4,204.115 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:196.19,198.13 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:209.111,213.4 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:224.139,227.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:229.122,232.3 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:234.69,235.32 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:235.32,238.19 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:242.4,252.14 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:238.19,240.13 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:252.14,259.65 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:264.5,264.99 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:280.5,283.66 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:288.5,291.65 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:259.65,261.6 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:264.99,270.160 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:270.160,272.52 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:272.52,275.8 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:283.66,285.6 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:291.65,293.6 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:299.12,302.3 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:305.43,309.20 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:311.14,315.32 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:362.3,362.54 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:364.45,370.19 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:315.32,318.19 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:322.4,337.59 4 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:344.4,350.69 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:355.4,359.135 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:318.19,320.13 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:337.59,342.5 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:350.69,352.5 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:375.104,385.55 4 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:394.2,394.22 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:399.2,399.76 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:433.2,433.16 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:385.55,386.152 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:386.152,388.122 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:388.122,390.5 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:394.22,396.3 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:399.76,400.51 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:405.3,405.36 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:408.3,408.34 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:400.51,401.141 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:401.141,403.5 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:405.36,407.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:410.8,410.92 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:410.92,411.51 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:417.3,419.20 3 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:422.3,422.34 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:428.3,428.38 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:431.3,431.36 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:411.51,412.141 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:412.141,414.5 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:419.20,421.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:422.34,424.37 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:424.37,426.5 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:428.38,430.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:436.88,445.111 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:450.2,450.39 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:466.2,466.49 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:522.2,522.14 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:445.111,447.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:450.39,452.18 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:456.3,460.51 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:452.18,455.4 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:460.51,463.4 2 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:466.49,467.46 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:470.3,472.63 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:467.46,469.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:472.63,474.4 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:475.8,475.47 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:475.47,476.37 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:480.3,488.70 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:492.3,492.67 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:476.37,478.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:488.70,490.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:492.67,494.4 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:494.9,494.74 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:494.74,496.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:498.8,498.53 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:498.53,499.37 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:503.3,511.70 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:515.3,515.67 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:499.37,501.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:511.70,513.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:515.67,517.4 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:517.9,517.74 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:517.74,519.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:525.62,533.50 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:538.2,538.21 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:543.2,545.14 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:533.50,534.36 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:534.36,536.4 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:538.21,542.3 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:548.58,557.56 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:563.2,563.97 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:571.2,571.41 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:576.2,576.13 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:557.56,560.3 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:563.97,564.45 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:564.45,566.187 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:566.187,568.5 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:571.41,574.3 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:579.80,581.37 2 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:592.2,592.25 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:597.2,597.33 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:601.2,601.23 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:581.37,582.46 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:582.46,584.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:584.9,586.4 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:587.8,589.3 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:592.25,594.3 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:597.33,599.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:604.81,614.2 6 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:616.49,623.32 4 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:623.32,625.3 1 0 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:625.8,631.3 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:635.77,638.38 3 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:643.2,643.12 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:638.38,639.89 1 1 -github.com/keikoproj/upgrade-manager/controllers/upgrade.go:639.89,641.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/script_runner.go:131.2,132.12 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:71.53,73.16 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:73.16,75.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:78.2,83.40 4 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:83.40,85.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:88.2,88.43 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:88.43,94.3 5 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:96.2,99.59 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:99.59,101.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:102.2,113.32 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:113.32,119.3 5 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:121.2,123.64 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:123.64,127.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:129.2,129.57 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:129.57,131.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:133.2,133.12 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:136.95,147.30 5 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:147.30,149.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:152.2,152.24 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:152.24,153.53 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:153.53,155.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:158.2,158.14 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:159.40,160.32 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:160.32,163.19 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:163.19,165.13 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:168.4,172.115 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:175.3,177.36 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:177.36,181.52 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:181.52,184.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:185.4,185.53 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:185.53,188.5 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:191.4,192.112 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:192.112,196.5 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:198.4,199.66 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:199.66,201.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:204.4,205.20 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:206.9,208.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:211.3,211.32 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:211.32,214.19 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:214.19,216.13 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:218.4,221.124 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:225.3,226.29 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:226.29,229.4 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:230.3,230.79 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:232.39,233.32 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:233.32,236.19 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:236.19,238.13 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:240.4,244.115 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:247.3,249.111 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:249.111,253.4 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:256.2,264.139 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:264.139,267.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:269.2,269.122 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:269.122,272.3 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:274.2,274.69 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:274.69,275.32 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:275.32,278.19 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:278.19,280.13 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:282.4,297.46 4 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:297.46,299.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:299.10,301.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:303.4,303.55 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:303.55,305.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:305.10,307.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:310.4,310.14 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:310.14,317.65 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:317.65,319.6 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:322.5,322.99 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:322.99,328.141 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:328.141,330.31 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:330.31,331.117 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:331.117,333.9 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:334.8,335.14 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:341.5,344.66 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:344.66,346.6 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:349.5,352.65 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:352.65,354.6 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:359.2,360.12 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:360.12,363.3 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:365.2,365.9 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:366.43,370.20 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:372.14,376.32 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:376.32,379.19 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:379.19,381.13 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:383.4,398.59 4 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:398.59,403.5 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:406.4,407.39 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:407.39,411.5 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:413.4,419.69 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:419.69,421.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:424.4,428.135 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:431.3,431.54 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:433.45,439.19 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:441.2,441.18 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:444.132,454.32 4 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:454.32,456.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:457.2,457.55 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:457.55,458.152 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:458.152,460.122 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:460.122,462.5 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:466.2,466.22 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:466.22,468.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:471.2,471.76 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:471.76,472.51 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:472.51,473.227 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:473.227,475.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:477.3,477.36 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:477.36,479.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:480.3,480.34 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:482.8,482.92 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:482.92,483.51 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:483.51,484.227 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:484.227,486.5 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:489.3,493.34 3 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:493.34,496.4 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:499.3,499.7 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:499.7,500.38 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:500.38,501.10 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:504.4,504.32 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:504.32,506.33 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:506.33,509.6 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:513.3,513.38 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:513.38,515.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:516.3,516.36 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:518.2,518.16 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:521.88,530.111 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:530.111,532.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:535.2,535.39 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:535.39,537.18 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:537.18,540.4 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:541.3,545.51 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:545.51,548.4 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:551.2,551.49 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:551.49,552.46 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:552.46,554.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:555.3,557.63 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:557.63,559.4 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:560.8,560.47 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:560.47,561.37 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:561.37,563.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:565.3,573.70 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:573.70,575.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:577.3,577.67 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:577.67,579.4 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:579.9,579.74 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:579.74,581.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:583.8,583.53 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:583.53,584.37 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:584.37,586.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:588.3,596.70 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:596.70,598.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:600.3,600.67 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:600.67,602.4 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:602.9,602.74 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:602.74,604.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:607.2,607.14 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:610.62,618.50 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:618.50,619.36 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:619.36,621.4 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:623.2,623.21 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:623.21,627.3 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:628.2,630.14 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:633.58,642.56 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:642.56,645.3 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:648.2,648.97 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:648.97,649.45 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:649.45,651.187 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:651.187,653.5 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:656.2,656.41 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:656.41,659.3 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:661.2,661.13 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:664.80,666.37 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:666.37,667.46 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:667.46,669.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:669.9,671.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:672.8,674.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:677.2,677.25 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:677.25,679.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:682.2,682.33 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:682.33,684.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:686.2,686.23 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:689.81,690.43 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:690.43,700.3 6 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:704.49,711.32 4 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:711.32,713.3 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:713.8,719.3 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:723.77,726.38 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:726.38,727.89 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:727.89,729.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:731.2,731.12 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:735.78,741.32 4 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:741.32,743.3 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:746.2,746.50 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:746.50,750.3 3 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:750.8,750.79 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:750.79,755.3 4 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:755.8,755.33 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:755.33,758.3 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:759.2,759.25 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:762.87,767.16 4 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:767.16,769.17 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:769.17,772.4 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:773.8,775.17 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:775.17,777.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:779.3,780.107 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:780.107,782.4 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:787.2,787.41 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:787.41,788.138 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:788.138,790.114 1 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:790.114,792.20 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:792.20,794.14 2 1 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:797.5,797.39 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:797.39,798.11 1 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:800.5,801.90 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:801.90,804.6 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:806.5,807.126 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:807.126,810.6 2 0 +github.com/keikoproj/upgrade-manager/controllers/upgrade.go:814.2,814.18 1 1 diff --git a/main.go b/main.go index dea54c74..6d3f3032 100644 --- a/main.go +++ b/main.go @@ -88,6 +88,7 @@ func main() { drainTimeout int ignoreDrainFailures bool maxReplacementNodes int + earlyCordonNodes bool ) flag.BoolVar(&debugMode, "debug", false, "enable debug logging") @@ -102,6 +103,7 @@ func main() { flag.IntVar(&drainTimeout, "drain-timeout", 900, "when the drain command should timeout") flag.BoolVar(&ignoreDrainFailures, "ignore-drain-failures", false, "proceed with instance termination despite drain failures.") flag.IntVar(&maxReplacementNodes, "max-replacement-nodes", 0, "The max number of replacement nodes allowed in a cluster. Avoids cluster-ballooning") + flag.BoolVar(&earlyCordonNodes, "early-cordon-nodes", false, "when enabled, will cordon all the nodes in the node-group even before processing the nodes") opts := zap.Options{ Development: true, @@ -210,6 +212,7 @@ func main() { DrainTimeout: drainTimeout, IgnoreDrainFailures: ignoreDrainFailures, ReplacementNodesMap: &sync.Map{}, + EarlyCordonNodes: earlyCordonNodes, } reconciler.SetMaxParallel(maxParallel)