diff --git a/pkg/karmadactl/addons/descheduler/descheduler.go b/pkg/karmadactl/addons/descheduler/descheduler.go index 255bce91019b..70bdd0eced02 100644 --- a/pkg/karmadactl/addons/descheduler/descheduler.go +++ b/pkg/karmadactl/addons/descheduler/descheduler.go @@ -13,13 +13,9 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" - "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" ) -var karmadaDeschedulerLabels = map[string]string{"app": addoninit.DeschedulerResourceName} - // AddonDescheduler describe the descheduler addon command process var AddonDescheduler = &addoninit.Addon{ Name: addoninit.DeschedulerResourceName, @@ -64,7 +60,7 @@ var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create karmada descheduler deployment error: %v", err) } - if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaDeschedulerLabels), opts.WaitPodReadyTimeout); err != nil { + if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada descheduler pod timeout: %v", err) } diff --git a/pkg/karmadactl/addons/enable.go b/pkg/karmadactl/addons/enable.go index afd7f15f548e..2513b6807b44 100644 --- a/pkg/karmadactl/addons/enable.go +++ b/pkg/karmadactl/addons/enable.go @@ -8,6 +8,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" "github.com/karmada-io/karmada/pkg/version" ) @@ -71,7 +72,7 @@ func NewCmdAddonsEnable(parentCommand string) *cobra.Command { flags := cmd.PersistentFlags() opts.GlobalCommandOptions.AddFlags(flags) - flags.IntVar(&opts.WaitPodReadyTimeout, "pod-timeout", 30, "Wait pod ready timeout.") + flags.IntVar(&opts.WaitComponentReadyTimeout, "pod-timeout", options.WaitComponentReadyTimeout, "Wait pod ready timeout.") flags.IntVar(&opts.WaitAPIServiceReadyTimeout, "apiservice-timeout", 30, "Wait apiservice ready timeout.") flags.StringVar(&opts.KarmadaSearchImage, "karmada-search-image", fmt.Sprintf("docker.io/karmada/karmada-search:%s", releaseVer.PatchRelease()), "karmada search image") flags.Int32Var(&opts.KarmadaSearchReplicas, "karmada-search-replicas", 1, "Karmada search replica set") diff --git a/pkg/karmadactl/addons/estimator/estimator.go b/pkg/karmadactl/addons/estimator/estimator.go index a55fdf10f9a9..d40725028684 100644 --- a/pkg/karmadactl/addons/estimator/estimator.go +++ b/pkg/karmadactl/addons/estimator/estimator.go @@ -15,8 +15,6 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" - "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -113,8 +111,7 @@ var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create or update scheduler estimator deployment error: %v", err) } - karmadaEstimatorLabels := map[string]string{"cluster": opts.Cluster} - if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaEstimatorLabels), opts.WaitPodReadyTimeout); err != nil { + if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster) diff --git a/pkg/karmadactl/addons/init/disable_option.go b/pkg/karmadactl/addons/init/disable_option.go index 0063c6784676..ad8a74613ba5 100644 --- a/pkg/karmadactl/addons/init/disable_option.go +++ b/pkg/karmadactl/addons/init/disable_option.go @@ -15,8 +15,6 @@ type CommandAddonsDisableOption struct { GlobalCommandOptions KarmadaKubeClientSet *kubernetes.Clientset - - WaitPodReadyTimeout int } // Complete the conditions required to be able to run disable. diff --git a/pkg/karmadactl/addons/init/enable_option.go b/pkg/karmadactl/addons/init/enable_option.go index 7bc1bdf1014d..7f7bfc7a7167 100644 --- a/pkg/karmadactl/addons/init/enable_option.go +++ b/pkg/karmadactl/addons/init/enable_option.go @@ -32,7 +32,7 @@ type CommandAddonsEnableOption struct { KarmadaKubeClientSet *kubernetes.Clientset - WaitPodReadyTimeout int + WaitComponentReadyTimeout int WaitAPIServiceReadyTimeout int diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go index 010da8d4a446..55cf15b835ea 100644 --- a/pkg/karmadactl/addons/search/search.go +++ b/pkg/karmadactl/addons/search/search.go @@ -19,8 +19,6 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" - "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" ) @@ -35,10 +33,6 @@ const ( etcdContainerClientPort = 2379 ) -var ( - karmadaSearchLabels = map[string]string{"app": addoninit.SearchResourceName, "apiserver": "true"} -) - // AddonSearch describe the search addon command process var AddonSearch = &addoninit.Addon{ Name: addoninit.SearchResourceName, @@ -166,7 +160,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada search deployment error: %v", err) } - if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaSearchLabels), opts.WaitPodReadyTimeout); err != nil { + if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada search pod status ready timeout: %v", err) } diff --git a/pkg/karmadactl/cmdinit/kubernetes/check.go b/pkg/karmadactl/cmdinit/kubernetes/check.go deleted file mode 100644 index fad18b01ba0e..000000000000 --- a/pkg/karmadactl/cmdinit/kubernetes/check.go +++ /dev/null @@ -1,107 +0,0 @@ -package kubernetes - -import ( - "context" - "fmt" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" -) - -func podStatus(pod *corev1.Pod) string { - for _, value := range pod.Status.ContainerStatuses { - if pod.Status.Phase == corev1.PodRunning { - if value.State.Waiting != nil { - return value.State.Waiting.Reason - } - if value.State.Waiting == nil { - return string(corev1.PodRunning) - } - return "Error" - } - if pod.ObjectMeta.DeletionTimestamp != nil { - return "Terminating" - } - } - return pod.Status.ContainerStatuses[0].State.Waiting.Reason -} - -func isPodReady(c kubernetes.Interface, n, p string) wait.ConditionFunc { - return func() (done bool, err error) { - pod, err := c.CoreV1().Pods(n).Get(context.TODO(), p, metav1.GetOptions{}) - if err != nil { - return false, err - } - - if pod.Status.Phase == corev1.PodPending && len(pod.Status.ContainerStatuses) == 0 { - klog.Warningf("Pod: %s not ready. status: %v", pod.Name, corev1.PodPending) - return false, nil - } - - for _, v := range pod.Status.Conditions { - switch v.Type { - case corev1.PodReady: - if v.Status == corev1.ConditionTrue { - klog.Infof("pod: %s is ready. status: %v", pod.Name, podStatus(pod)) - return true, nil - } - klog.Warningf("Pod: %s not ready. status: %v", pod.Name, podStatus(pod)) - return false, nil - default: - continue - } - } - return false, err - } -} - -// waitPodReady Poll up to timeout seconds for pod to enter running state. -// Returns an error if the pod never enters the running state. -func waitPodReady(c kubernetes.Interface, namespaces, podName string, timeout time.Duration) error { - return wait.PollImmediate(time.Second, timeout, isPodReady(c, namespaces, podName)) -} - -// WaitPodReady wait pod ready -func WaitPodReady(c kubernetes.Interface, namespace, selector string, timeout int) error { - // Wait 3 second - time.Sleep(3 * time.Second) - pods, err := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector}) - if err != nil { - return err - } - - if len(pods.Items) == 0 { - return fmt.Errorf("no pods in %s with selector %s", namespace, selector) - } - - for _, pod := range pods.Items { - if err = waitPodReady(c, namespace, pod.Name, time.Duration(timeout)*time.Second); err != nil { - return err - } - } - - return nil -} - -// WaitEtcdReplicasetInDesired Wait Etcd Ready -func WaitEtcdReplicasetInDesired(replicas int32, c kubernetes.Interface, namespace, selector string, timeout int) error { - if err := wait.PollImmediate(time.Second, time.Duration(timeout)*time.Second, func() (done bool, err error) { - pods, e := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector}) - if e != nil { - return false, nil - } - if int32(len(pods.Items)) == replicas { - klog.Infof("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items)) - return true, nil - } - klog.Warningf("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items)) - return false, nil - }); err != nil { - return err - } - return nil -} diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index bae675ca33d5..1ddb865fabd4 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -347,28 +347,30 @@ func (i *CommandInitOption) createCertsSecrets() error { } func (i *CommandInitOption) initKarmadaAPIServer() error { + // wait karmada APIServer component ready timeout 120s + waitKarmadaAPIServerComponentReadyTimeout := 120 + if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil { return err } klog.Info("Create etcd StatefulSets") - if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), i.makeETCDStatefulSet(), metav1.CreateOptions{}); err != nil { - klog.Warning(err) - } - if err := WaitEtcdReplicasetInDesired(i.EtcdReplicas, i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil { + etcdStatefulSet := i.makeETCDStatefulSet() + if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), etcdStatefulSet, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil { + if err := util.WaitForStatefulSetRollout(i.KubeClientSet, etcdStatefulSet, options.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } - klog.Info("Create karmada ApiServer Deployment") if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil { return err } - if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAPIServerDeployment(), metav1.CreateOptions{}); err != nil { + + karmadaAPIServerDeployment := i.makeKarmadaAPIServerDeployment() + if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAPIServerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(apiServerLabels), 120); err != nil { + if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAPIServerDeployment, waitKarmadaAPIServerComponentReadyTimeout); err != nil { return err } @@ -378,19 +380,17 @@ func (i *CommandInitOption) initKarmadaAPIServer() error { if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaAggregatedAPIServerService()); err != nil { klog.Exitln(err) } + karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment() if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(aggregatedAPIServerLabels), 30); err != nil { + if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, options.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } return nil } func (i *CommandInitOption) initKarmadaComponent() error { - // wait pod ready timeout 30s - waitPodReadyTimeout := 30 - deploymentClient := i.KubeClientSet.AppsV1().Deployments(i.Namespace) // Create karmada-kube-controller-manager // https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/kube-controller-manager.yaml @@ -398,30 +398,33 @@ func (i *CommandInitOption) initKarmadaComponent() error { if err := util.CreateOrUpdateService(i.KubeClientSet, i.kubeControllerManagerService()); err != nil { klog.Exitln(err) } - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaKubeControllerManagerDeployment(), metav1.CreateOptions{}); err != nil { + karmadaKubeControllerManagerDeployment := i.makeKarmadaKubeControllerManagerDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaKubeControllerManagerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(kubeControllerManagerLabels), waitPodReadyTimeout); err != nil { + if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaKubeControllerManagerDeployment, options.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } // Create karmada-scheduler // https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-scheduler.yaml klog.Info("Create karmada scheduler Deployment") - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaSchedulerDeployment(), metav1.CreateOptions{}); err != nil { + karmadaSchedulerDeployment := i.makeKarmadaSchedulerDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaSchedulerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(schedulerLabels), waitPodReadyTimeout); err != nil { + if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaSchedulerDeployment, options.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } // Create karmada-controller-manager // https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-controller-manager.yaml klog.Info("Create karmada controller manager Deployment") - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaControllerManagerDeployment(), metav1.CreateOptions{}); err != nil { + karmadaControllerManagerDeployment := i.makeKarmadaControllerManagerDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaControllerManagerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(controllerManagerLabels), waitPodReadyTimeout); err != nil { + if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaControllerManagerDeployment, options.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } @@ -431,10 +434,11 @@ func (i *CommandInitOption) initKarmadaComponent() error { if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaWebhookService()); err != nil { klog.Exitln(err) } - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaWebhookDeployment(), metav1.CreateOptions{}); err != nil { + karmadaWebhookDeployment := i.makeKarmadaWebhookDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaWebhookDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(webhookLabels), waitPodReadyTimeout); err != nil { + if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaWebhookDeployment, options.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } return nil diff --git a/pkg/karmadactl/cmdinit/options/global.go b/pkg/karmadactl/cmdinit/options/global.go index 7b296b064ac3..b47fcc7294ac 100644 --- a/pkg/karmadactl/cmdinit/options/global.go +++ b/pkg/karmadactl/cmdinit/options/global.go @@ -23,4 +23,6 @@ const ( UserName = "karmada-admin" // KarmadaKubeConfigName karmada kubeconfig name KarmadaKubeConfigName = "karmada-apiserver.config" + // WaitComponentReadyTimeout wait component ready time + WaitComponentReadyTimeout = 30 ) diff --git a/pkg/karmadactl/register/register.go b/pkg/karmadactl/register/register.go index 8fd3063f1c89..1bacc49d1d4a 100644 --- a/pkg/karmadactl/register/register.go +++ b/pkg/karmadactl/register/register.go @@ -34,8 +34,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/karmada-io/karmada/pkg/apis/cluster/validation" - check "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -349,11 +347,12 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { // create karmada-agent Deployment in the member cluster fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment") - if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), o.makeKarmadaAgentDeployment(), metav1.CreateOptions{}); err != nil { + KarmadaAgentDeployment := o.makeKarmadaAgentDeployment() + if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { return err } - if err := check.WaitPodReady(o.memberClusterClient, o.Namespace, utils.MapToString(karmadaAgentLabels), int(o.Timeout)); err != nil { + if err := cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err } diff --git a/pkg/karmadactl/util/check.go b/pkg/karmadactl/util/check.go new file mode 100644 index 000000000000..5152fba3e737 --- /dev/null +++ b/pkg/karmadactl/util/check.go @@ -0,0 +1,77 @@ +package util + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +// WaitForStatefulSetRollout wait for StatefulSet reaches the ready state or timeout. +func WaitForStatefulSetRollout(c kubernetes.Interface, sts *appsv1.StatefulSet, timeoutSeconds int) error { + var lastErr error + pollError := wait.PollImmediate(time.Second, time.Duration(timeoutSeconds)*time.Second, func() (bool, error) { + s, err := c.AppsV1().StatefulSets(sts.GetNamespace()).Get(context.TODO(), sts.GetName(), metav1.GetOptions{}) + if err != nil { + lastErr = err + return false, nil + } + if s.Generation != s.Status.ObservedGeneration { + lastErr = fmt.Errorf("expected generation %d, observed generation: %d", + s.Generation, s.Status.ObservedGeneration) + return false, nil + } + if (s.Spec.Replicas != nil) && (s.Status.UpdatedReplicas < *s.Spec.Replicas) { + lastErr = fmt.Errorf("expected %d replicas, got %d updated replicas", + *s.Spec.Replicas, s.Status.UpdatedReplicas) + return false, nil + } + if s.Status.AvailableReplicas < s.Status.UpdatedReplicas { + lastErr = fmt.Errorf("expected %d replicas, got %d available replicas", + s.Status.UpdatedReplicas, s.Status.AvailableReplicas) + return false, nil + } + return true, nil + }) + if pollError != nil { + return fmt.Errorf("wait for Statefulset(%s/%s) rollout: %v: %v", sts.GetNamespace(), sts.GetName(), pollError, lastErr) + } + return nil +} + +// WaitForDeploymentRollout wait for Deployment reaches the ready state or timeout. +func WaitForDeploymentRollout(c kubernetes.Interface, dep *appsv1.Deployment, timeoutSeconds int) error { + var lastErr error + pollError := wait.PollImmediate(time.Second, time.Duration(timeoutSeconds)*time.Second, func() (bool, error) { + d, err := c.AppsV1().Deployments(dep.GetNamespace()).Get(context.TODO(), dep.GetName(), metav1.GetOptions{}) + if err != nil { + lastErr = err + return false, nil + } + if d.Generation != d.Status.ObservedGeneration { + lastErr = fmt.Errorf("current generation %d, observed generation %d", + d.Generation, d.Status.ObservedGeneration) + return false, nil + } + if (d.Spec.Replicas != nil) && (d.Status.UpdatedReplicas < *d.Spec.Replicas) { + lastErr = fmt.Errorf("the number of pods targeted by the deployment (%d pods) is different "+ + "from the number of pods targeted by the deployment that have the desired template spec (%d pods)", + *d.Spec.Replicas, d.Status.UpdatedReplicas) + return false, nil + } + if d.Status.AvailableReplicas < d.Status.UpdatedReplicas { + lastErr = fmt.Errorf("expected %d replicas, got %d available replicas", + d.Status.UpdatedReplicas, d.Status.AvailableReplicas) + return false, nil + } + return true, nil + }) + if pollError != nil { + return fmt.Errorf("wait for Deployment(%s/%s) rollout: %v: %v", dep.GetNamespace(), dep.GetName(), pollError, lastErr) + } + return nil +}