check deployment statefulset rollout not use check pod ready

Signed-off-by: huangyanfeng <[email protected]>

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.7.0
 	github.com/onsi/gomega v1.25.0
 	github.com/opensearch-project/opensearch-go v1.1.0
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
@@ -125,7 +126,6 @@ require (
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect

pkg/karmadactl/addons/descheduler/descheduler.go

@@ -14,12 +14,9 @@ import (
 	addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init"
 	addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
 	"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
-	initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
 	cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
 )
 
-var karmadaDeschedulerLabels = map[string]string{"app": addoninit.DeschedulerResourceName}
-
 // AddonDescheduler describe the descheduler addon command process
 var AddonDescheduler = &addoninit.Addon{
 	Name:    addoninit.DeschedulerResourceName,
@@ -64,7 +61,7 @@ var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error {
 		return fmt.Errorf("create karmada descheduler deployment error: %v", err)
 	}
 
-	if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaDeschedulerLabels), opts.WaitPodReadyTimeout); err != nil {
+	if err := kubernetes.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitPodReadyTimeout); err != nil {
 		return fmt.Errorf("wait karmada descheduler pod timeout: %v", err)
 	}
 

pkg/karmadactl/addons/estimator/estimator.go

@@ -16,7 +16,6 @@ import (
 	addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init"
 	addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
 	"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
-	initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
 	cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
 	"github.com/karmada-io/karmada/pkg/util/names"
 )
@@ -113,8 +112,7 @@ var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error {
 		return fmt.Errorf("create or update scheduler estimator deployment error: %v", err)
 	}
 
-	karmadaEstimatorLabels := map[string]string{"cluster": opts.Cluster}
-	if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaEstimatorLabels), opts.WaitPodReadyTimeout); err != nil {
+	if err := kubernetes.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitPodReadyTimeout); err != nil {
 		klog.Warning(err)
 	}
 	klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster)

pkg/karmadactl/addons/search/search.go

@@ -20,7 +20,6 @@ import (
 	addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
 	initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada"
 	"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
-	initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
 	cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
 )
 
@@ -35,10 +34,6 @@ const (
 	etcdContainerClientPort = 2379
 )
 
-var (
-	karmadaSearchLabels = map[string]string{"app": addoninit.SearchResourceName, "apiserver": "true"}
-)
-
 // AddonSearch describe the search addon command process
 var AddonSearch = &addoninit.Addon{
 	Name:    addoninit.SearchResourceName,
@@ -166,7 +161,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e
 		return fmt.Errorf("create karmada search deployment error: %v", err)
 	}
 
-	if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaSearchLabels), opts.WaitPodReadyTimeout); err != nil {
+	if err := kubernetes.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitPodReadyTimeout); err != nil {
 		return fmt.Errorf("wait karmada search pod status ready timeout: %v", err)
 	}
 

pkg/karmadactl/cmdinit/kubernetes/check.go

@@ -2,106 +2,83 @@ package kubernetes
 
 import (
 	"context"
-	"fmt"
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 )
 
-func podStatus(pod *corev1.Pod) string {
-	for _, value := range pod.Status.ContainerStatuses {
-		if pod.Status.Phase == corev1.PodRunning {
-			if value.State.Waiting != nil {
-				return value.State.Waiting.Reason
-			}
-			if value.State.Waiting == nil {
-				return string(corev1.PodRunning)
-			}
-			return "Error"
-		}
-		if pod.ObjectMeta.DeletionTimestamp != nil {
-			return "Terminating"
-		}
-	}
-	return pod.Status.ContainerStatuses[0].State.Waiting.Reason
-}
-
-func isPodReady(c kubernetes.Interface, n, p string) wait.ConditionFunc {
-	return func() (done bool, err error) {
-		pod, err := c.CoreV1().Pods(n).Get(context.TODO(), p, metav1.GetOptions{})
+// WaitForStatefulsetRollout wait statefulset rollout
+func WaitForStatefulsetRollout(c kubernetes.Interface, sts *appsv1.StatefulSet, timeout int) error {
+	var lastErr error
+	if err := wait.Poll(time.Second, time.Duration(timeout)*time.Second, func() (bool, error) {
+		s, err := c.AppsV1().StatefulSets(sts.GetNamespace()).Get(context.TODO(), sts.GetName(), metav1.GetOptions{})
 		if err != nil {
-			return false, err
+			lastErr = err
+			klog.V(4).ErrorS(err, "WaitForStatefulsetRollout: failed to get StatefulSet")
+			return false, nil
 		}
-
-		if pod.Status.Phase == corev1.PodPending && len(pod.Status.ContainerStatuses) == 0 {
-			klog.Warningf("Pod: %s not ready. status: %v", pod.Name, corev1.PodPending)
+		if s.Generation > s.Status.ObservedGeneration {
+			lastErr = errors.Errorf("expected generation %d, observed generation: %d",
+				s.Generation, s.Status.ObservedGeneration)
 			return false, nil
 		}
-
-		for _, v := range pod.Status.Conditions {
-			switch v.Type {
-			case corev1.PodReady:
-				if v.Status == corev1.ConditionTrue {
-					klog.Infof("pod: %s is ready. status: %v", pod.Name, podStatus(pod))
-					return true, nil
-				}
-				klog.Warningf("Pod: %s not ready. status: %v", pod.Name, podStatus(pod))
-				return false, nil
-			default:
-				continue
-			}
+		if s.Status.UpdatedReplicas != s.Status.Replicas {
+			lastErr = errors.Errorf("expected %d replicas, got %d updated replicas",
+				s.Status.Replicas, s.Status.UpdatedReplicas)
+			return false, nil
 		}
-		return false, err
-	}
-}
-
-// waitPodReady  Poll up to timeout seconds for pod to enter running state.
-// Returns an error if the pod never enters the running state.
-func waitPodReady(c kubernetes.Interface, namespaces, podName string, timeout time.Duration) error {
-	return wait.PollImmediate(time.Second, timeout, isPodReady(c, namespaces, podName))
-}
-
-// WaitPodReady wait pod ready
-func WaitPodReady(c kubernetes.Interface, namespace, selector string, timeout int) error {
-	// Wait 3 second
-	time.Sleep(3 * time.Second)
-	pods, err := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector})
-	if err != nil {
-		return err
-	}
-
-	if len(pods.Items) == 0 {
-		return fmt.Errorf("no pods in %s with selector %s", namespace, selector)
-	}
-
-	for _, pod := range pods.Items {
-		if err = waitPodReady(c, namespace, pod.Name, time.Duration(timeout)*time.Second); err != nil {
-			return err
+		if s.Status.ReadyReplicas != s.Status.Replicas {
+			lastErr = errors.Errorf("expected %d replicas, got %d ready replicas",
+				s.Status.Replicas, s.Status.ReadyReplicas)
+			return false, nil
 		}
+		return true, nil
+	}); err != nil {
+		if err == wait.ErrWaitTimeout && lastErr != nil {
+			err = lastErr
+		}
+		return errors.Wrapf(err, "waiting for StatefulsetRollout of %s/%s", sts.GetNamespace(), sts.GetName())
 	}
-
 	return nil
 }
 
-// WaitEtcdReplicasetInDesired Wait Etcd Ready
-func WaitEtcdReplicasetInDesired(replicas int32, c kubernetes.Interface, namespace, selector string, timeout int) error {
-	if err := wait.PollImmediate(time.Second, time.Duration(timeout)*time.Second, func() (done bool, err error) {
-		pods, e := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector})
-		if e != nil {
+// WaitForDeploymentRollout wait deployment rollout
+func WaitForDeploymentRollout(c kubernetes.Interface, dep *appsv1.Deployment, timeout int) error {
+	var lastErr error
+	if err := wait.Poll(time.Second, time.Duration(timeout)*time.Second, func() (bool, error) {
+		d, err := c.AppsV1().Deployments(dep.GetNamespace()).Get(context.TODO(), dep.GetName(), metav1.GetOptions{})
+		if err != nil {
+			lastErr = err
+			klog.V(4).ErrorS(err, "WaitForDeploymentRollout: failed to get Deployment")
+			return false, nil
+		}
+		if d.Generation > d.Status.ObservedGeneration {
+			lastErr = errors.Errorf("current generation %d, observed generation %d",
+				d.Generation, d.Status.ObservedGeneration)
 			return false, nil
 		}
-		if int32(len(pods.Items)) == replicas {
-			klog.Infof("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items))
-			return true, nil
+		if d.Status.UpdatedReplicas != d.Status.Replicas {
+			lastErr = errors.Errorf("the number of pods targeted by the deployment (%d pods) is different "+
+				"from the number of pods targeted by the deployment that have the desired template spec (%d pods)",
+				d.Status.Replicas, d.Status.UpdatedReplicas)
+			return false, nil
 		}
-		klog.Warningf("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items))
-		return false, nil
+		if d.Status.UnavailableReplicas != 0 {
+			lastErr = errors.Errorf("got %d unavailable replicas",
+				d.Status.UnavailableReplicas)
+			return false, nil
+		}
+		return true, nil
 	}); err != nil {
-		return err
+		if err == wait.ErrWaitTimeout && lastErr != nil {
+			err = lastErr
+		}
+		return errors.Wrapf(err, "waiting for DeploymentRollout of %s/%s", dep.GetNamespace(), dep.GetName())
 	}
 	return nil
 }

pkg/karmadactl/cmdinit/kubernetes/deploy.go

@@ -351,24 +351,23 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
 		return err
 	}
 	klog.Info("Create etcd StatefulSets")
-	if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), i.makeETCDStatefulSet(), metav1.CreateOptions{}); err != nil {
+	etcdStatefulSet := i.makeETCDStatefulSet()
+	if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), etcdStatefulSet, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitEtcdReplicasetInDesired(i.EtcdReplicas, i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil {
+	if err := WaitForStatefulsetRollout(i.KubeClientSet, etcdStatefulSet, 30); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil {
-		klog.Warning(err)
-	}
-
 	klog.Info("Create karmada ApiServer Deployment")
 	if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil {
 		return err
 	}
-	if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAPIServerDeployment(), metav1.CreateOptions{}); err != nil {
+
+	karmadaAPIServerDeployment := i.makeKarmadaAPIServerDeployment()
+	if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAPIServerDeployment, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(apiServerLabels), 120); err != nil {
+	if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaAPIServerDeployment, 120); err != nil {
 		return err
 	}
 
@@ -378,10 +377,11 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
 	if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaAggregatedAPIServerService()); err != nil {
 		klog.Exitln(err)
 	}
+	karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment()
 	if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(aggregatedAPIServerLabels), 30); err != nil {
+	if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, 30); err != nil {
 		klog.Warning(err)
 	}
 	return nil
@@ -398,30 +398,33 @@ func (i *CommandInitOption) initKarmadaComponent() error {
 	if err := util.CreateOrUpdateService(i.KubeClientSet, i.kubeControllerManagerService()); err != nil {
 		klog.Exitln(err)
 	}
-	if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaKubeControllerManagerDeployment(), metav1.CreateOptions{}); err != nil {
+	karmadaKubeControllerManagerDeployment := i.makeKarmadaKubeControllerManagerDeployment()
+	if _, err := deploymentClient.Create(context.TODO(), karmadaKubeControllerManagerDeployment, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(kubeControllerManagerLabels), waitPodReadyTimeout); err != nil {
+	if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaKubeControllerManagerDeployment, waitPodReadyTimeout); err != nil {
 		klog.Warning(err)
 	}
 
 	// Create karmada-scheduler
 	// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-scheduler.yaml
 	klog.Info("Create karmada scheduler Deployment")
-	if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaSchedulerDeployment(), metav1.CreateOptions{}); err != nil {
+	karmadaSchedulerDeployment := i.makeKarmadaSchedulerDeployment()
+	if _, err := deploymentClient.Create(context.TODO(), karmadaSchedulerDeployment, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(schedulerLabels), waitPodReadyTimeout); err != nil {
+	if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaSchedulerDeployment, waitPodReadyTimeout); err != nil {
 		klog.Warning(err)
 	}
 
 	// Create karmada-controller-manager
 	// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-controller-manager.yaml
 	klog.Info("Create karmada controller manager Deployment")
-	if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaControllerManagerDeployment(), metav1.CreateOptions{}); err != nil {
+	karmadaControllerManagerDeployment := i.makeKarmadaControllerManagerDeployment()
+	if _, err := deploymentClient.Create(context.TODO(), karmadaControllerManagerDeployment, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(controllerManagerLabels), waitPodReadyTimeout); err != nil {
+	if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaControllerManagerDeployment, waitPodReadyTimeout); err != nil {
 		klog.Warning(err)
 	}
 
@@ -431,10 +434,11 @@ func (i *CommandInitOption) initKarmadaComponent() error {
 	if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaWebhookService()); err != nil {
 		klog.Exitln(err)
 	}
-	if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaWebhookDeployment(), metav1.CreateOptions{}); err != nil {
+	karmadaWebhookDeployment := i.makeKarmadaWebhookDeployment()
+	if _, err := deploymentClient.Create(context.TODO(), karmadaWebhookDeployment, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
-	if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(webhookLabels), waitPodReadyTimeout); err != nil {
+	if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaWebhookDeployment, waitPodReadyTimeout); err != nil {
 		klog.Warning(err)
 	}
 	return nil

pkg/karmadactl/register/register.go

@@ -35,7 +35,6 @@ import (
 
 	"github.com/karmada-io/karmada/pkg/apis/cluster/validation"
 	check "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
-	"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
 	"github.com/karmada-io/karmada/pkg/karmadactl/options"
 	cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
 	"github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient"
@@ -349,11 +348,12 @@ func (o *CommandRegisterOption) Run(parentCommand string) error {
 
 	// create karmada-agent Deployment in the member cluster
 	fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment")
-	if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), o.makeKarmadaAgentDeployment(), metav1.CreateOptions{}); err != nil {
+	KarmadaAgentDeployment := o.makeKarmadaAgentDeployment()
+	if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
-	if err := check.WaitPodReady(o.memberClusterClient, o.Namespace, utils.MapToString(karmadaAgentLabels), int(o.Timeout)); err != nil {
+	if err := check.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil {
 		return err
 	}