diff --git a/test/internal/kubernetes/utils.go b/test/internal/kubernetes/utils.go index b04f1f2772..caefd465ab 100644 --- a/test/internal/kubernetes/utils.go +++ b/test/internal/kubernetes/utils.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/test/internal/retry" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -33,8 +34,10 @@ const ( SubnetNameLabel = "kubernetes.azure.com/podnetwork-subnet" // RetryAttempts is the number of times to retry a test. - RetryAttempts = 90 - RetryDelay = 10 * time.Second + RetryAttempts = 90 + RetryDelay = 10 * time.Second + DeleteRetryAttempts = 12 + DeleteRetryDelay = 5 * time.Second ) var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file") @@ -247,6 +250,19 @@ func WaitForPodDeployment(ctx context.Context, clientset *kubernetes.Clientset, return errors.Wrapf(retrier.Do(ctx, checkPodDeploymentFn), "could not wait for deployment %s", deploymentName) } +func WaitForDeploymentToDelete(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, d appsv1.Deployment) error { + assertDeploymentNotFound := func() error { + _, err := deploymentsClient.Get(ctx, d.Name, metav1.GetOptions{}) + // only if the error is "isNotFound", do we say, the deployment is deleted + if apierrors.IsNotFound(err) { + return nil + } + return errors.Errorf(fmt.Sprintf("expected isNotFound error when getting deployment, but got %+v", err)) + } + retrier := retry.Retrier{Attempts: DeleteRetryAttempts, Delay: DeleteRetryDelay} + return errors.Wrapf(retrier.Do(ctx, assertDeploymentNotFound), "could not assert deployment %s isNotFound", d.Name) +} + func WaitForPodDaemonset(ctx context.Context, clientset *kubernetes.Clientset, namespace, daemonsetName, podLabelSelector string) error { podsClient := clientset.CoreV1().Pods(namespace) daemonsetClient := clientset.AppsV1().DaemonSets(namespace) diff --git a/test/internal/kubernetes/utils_delete.go b/test/internal/kubernetes/utils_delete.go index d2ef27926f..acbdf8a375 100644 --- a/test/internal/kubernetes/utils_delete.go +++ b/test/internal/kubernetes/utils_delete.go @@ -35,6 +35,9 @@ func MustDeleteDeployment(ctx context.Context, deployments typedappsv1.Deploymen panic(errors.Wrap(err, "failed to delete deployment")) } } + if err := WaitForDeploymentToDelete(ctx, deployments, d); err != nil { + panic(errors.Wrap(err, "failed to wait for deployment to delete")) + } } func MustDeleteNamespace(ctx context.Context, clienset *kubernetes.Clientset, namespace string) {