Make reset a bit more robust

* Retry failed operations * Don't error out on not found some of the objects that can be deleted by controllers (avoid race-condition) * Log a message to try to use --destroy-workers=false flag Signed-off-by: Artiom Diomin <[email protected]>
kubermatic · Jun 23, 2019 · 60ec5b1 · 60ec5b1
1 parent bdc7dfc
commit 60ec5b1
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 8 deletions.
diff --git a/pkg/installer/installation/reset.go b/pkg/installer/installation/reset.go
@@ -17,12 +17,16 @@ limitations under the License.
 package installation
 
 import (
+	"time"
+
 	"github.com/pkg/errors"
 
 	kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone"
 	"github.com/kubermatic/kubeone/pkg/ssh"
 	"github.com/kubermatic/kubeone/pkg/templates/machinecontroller"
 	"github.com/kubermatic/kubeone/pkg/util"
+
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // Reset undos all changes made by KubeOne to the configured machines.
@@ -51,14 +55,21 @@ func Reset(ctx *util.Context) error {
 func destroyWorkers(ctx *util.Context) error {
 	ctx.Logger.Infoln("Destroying worker nodes…")
 
-	if err := util.BuildKubernetesClientset(ctx); err != nil {
-		return errors.Wrap(err, "unable to build kubernetes clientset")
-	}
-	if err := machinecontroller.DestroyWorkers(ctx); err != nil {
-		return errors.Wrap(err, "unable to delete all worker nodes")
+	waitErr := wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
+		err := util.BuildKubernetesClientset(ctx)
+		return err == nil, errors.Wrap(err, "unable to build kubernetes clientset")
+	})
+	if waitErr != nil {
+		ctx.Logger.Info("try to use `--destroy-workers=false` flag")
+		return waitErr
 	}
 
-	return nil
+	waitErr = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
+		err := machinecontroller.DestroyWorkers(ctx)
+		return err == nil, errors.Wrap(err, "unable to delete all worker nodes")
+	})
+
+	return waitErr
 }
 
 func resetNode(ctx *util.Context, _ *kubeoneapi.HostConfig, conn ssh.Connection) error {
@@ -151,3 +162,15 @@ sudo rm /etc/kubernetes/cloud-config
 rm -rf "{{ .WORK_DIR }}"
 `
 )
+
+func defaultRetryBackoff(retries int) wait.Backoff {
+	if retries == 0 {
+		retries = 1
+	}
+
+	return wait.Backoff{
+		Steps:    retries,
+		Duration: 5 * time.Second,
+		Factor:   2.0,
+	}
+}
diff --git a/pkg/templates/machinecontroller/helper.go b/pkg/templates/machinecontroller/helper.go
@@ -146,7 +146,9 @@ func DestroyWorkers(ctx *util.Context) error {
 	}
 	for i := range msList.Items {
 		if err := ctx.DynamicClient.Delete(bgCtx, &msList.Items[i]); err != nil {
-			return errors.Wrapf(err, "unable to delete machineset object %s", msList.Items[i].Name)
+			if !errorsutil.IsNotFound(err) {
+				return errors.Wrapf(err, "unable to delete machineset object %s", msList.Items[i].Name)
+			}
 		}
 	}
 
@@ -160,7 +162,9 @@ func DestroyWorkers(ctx *util.Context) error {
 	}
 	for i := range mList.Items {
 		if err := ctx.DynamicClient.Delete(bgCtx, &mList.Items[i]); err != nil {
-			return errors.Wrapf(err, "unable to delete machine object %s", mList.Items[i].Name)
+			if !errorsutil.IsNotFound(err) {
+				return errors.Wrapf(err, "unable to delete machine object %s", mList.Items[i].Name)
+			}
 		}
 	}