diff --git a/pkg/installer/installation/reset.go b/pkg/installer/installation/reset.go index 7706fe3c6..9c7b68623 100644 --- a/pkg/installer/installation/reset.go +++ b/pkg/installer/installation/reset.go @@ -17,12 +17,16 @@ limitations under the License. package installation import ( + "time" + "github.com/pkg/errors" kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone" "github.com/kubermatic/kubeone/pkg/ssh" "github.com/kubermatic/kubeone/pkg/templates/machinecontroller" "github.com/kubermatic/kubeone/pkg/util" + + "k8s.io/apimachinery/pkg/util/wait" ) // Reset undos all changes made by KubeOne to the configured machines. @@ -51,14 +55,21 @@ func Reset(ctx *util.Context) error { func destroyWorkers(ctx *util.Context) error { ctx.Logger.Infoln("Destroying worker nodes…") - if err := util.BuildKubernetesClientset(ctx); err != nil { - return errors.Wrap(err, "unable to build kubernetes clientset") - } - if err := machinecontroller.DestroyWorkers(ctx); err != nil { - return errors.Wrap(err, "unable to delete all worker nodes") + waitErr := wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { + err := util.BuildKubernetesClientset(ctx) + return err == nil, errors.Wrap(err, "unable to build kubernetes clientset") + }) + if waitErr != nil { + ctx.Logger.Info("try to use `--destroy-workers=false` flag") + return waitErr } - return nil + waitErr = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { + err := machinecontroller.DestroyWorkers(ctx) + return err == nil, errors.Wrap(err, "unable to delete all worker nodes") + }) + + return waitErr } func resetNode(ctx *util.Context, _ *kubeoneapi.HostConfig, conn ssh.Connection) error { @@ -151,3 +162,15 @@ sudo rm /etc/kubernetes/cloud-config rm -rf "{{ .WORK_DIR }}" ` ) + +func defaultRetryBackoff(retries int) wait.Backoff { + if retries == 0 { + retries = 1 + } + + return wait.Backoff{ + Steps: retries, + Duration: 5 * time.Second, + Factor: 2.0, + } +} diff --git a/pkg/templates/machinecontroller/helper.go b/pkg/templates/machinecontroller/helper.go index c7e365550..bd2854276 100644 --- a/pkg/templates/machinecontroller/helper.go +++ b/pkg/templates/machinecontroller/helper.go @@ -146,7 +146,9 @@ func DestroyWorkers(ctx *util.Context) error { } for i := range msList.Items { if err := ctx.DynamicClient.Delete(bgCtx, &msList.Items[i]); err != nil { - return errors.Wrapf(err, "unable to delete machineset object %s", msList.Items[i].Name) + if !errorsutil.IsNotFound(err) { + return errors.Wrapf(err, "unable to delete machineset object %s", msList.Items[i].Name) + } } } @@ -160,7 +162,9 @@ func DestroyWorkers(ctx *util.Context) error { } for i := range mList.Items { if err := ctx.DynamicClient.Delete(bgCtx, &mList.Items[i]); err != nil { - return errors.Wrapf(err, "unable to delete machine object %s", mList.Items[i].Name) + if !errorsutil.IsNotFound(err) { + return errors.Wrapf(err, "unable to delete machine object %s", mList.Items[i].Name) + } } }