diff --git a/go.mod b/go.mod index e23f5ff53525..77d94e557d12 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/onsi/ginkgo/v2 v2.7.0 github.com/onsi/gomega v1.25.0 github.com/opensearch-project/opensearch-go v1.1.0 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 @@ -125,7 +126,6 @@ require ( github.com/pelletier/go-toml v1.9.4 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/pkg/karmadactl/addons/descheduler/descheduler.go b/pkg/karmadactl/addons/descheduler/descheduler.go index 255bce91019b..52c2e30f4ef8 100644 --- a/pkg/karmadactl/addons/descheduler/descheduler.go +++ b/pkg/karmadactl/addons/descheduler/descheduler.go @@ -14,12 +14,9 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" ) -var karmadaDeschedulerLabels = map[string]string{"app": addoninit.DeschedulerResourceName} - // AddonDescheduler describe the descheduler addon command process var AddonDescheduler = &addoninit.Addon{ Name: addoninit.DeschedulerResourceName, @@ -64,7 +61,7 @@ var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create karmada descheduler deployment error: %v", err) } - if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaDeschedulerLabels), opts.WaitPodReadyTimeout); err != nil { + if err := kubernetes.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitPodReadyTimeout); err != nil { return fmt.Errorf("wait karmada descheduler pod timeout: %v", err) } diff --git a/pkg/karmadactl/addons/estimator/estimator.go b/pkg/karmadactl/addons/estimator/estimator.go index a55fdf10f9a9..383323a40fe4 100644 --- a/pkg/karmadactl/addons/estimator/estimator.go +++ b/pkg/karmadactl/addons/estimator/estimator.go @@ -16,7 +16,6 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -113,8 +112,7 @@ var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create or update scheduler estimator deployment error: %v", err) } - karmadaEstimatorLabels := map[string]string{"cluster": opts.Cluster} - if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaEstimatorLabels), opts.WaitPodReadyTimeout); err != nil { + if err := kubernetes.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitPodReadyTimeout); err != nil { klog.Warning(err) } klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster) diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go index 010da8d4a446..4d85bee32637 100644 --- a/pkg/karmadactl/addons/search/search.go +++ b/pkg/karmadactl/addons/search/search.go @@ -20,7 +20,6 @@ import ( addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" ) @@ -35,10 +34,6 @@ const ( etcdContainerClientPort = 2379 ) -var ( - karmadaSearchLabels = map[string]string{"app": addoninit.SearchResourceName, "apiserver": "true"} -) - // AddonSearch describe the search addon command process var AddonSearch = &addoninit.Addon{ Name: addoninit.SearchResourceName, @@ -166,7 +161,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada search deployment error: %v", err) } - if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaSearchLabels), opts.WaitPodReadyTimeout); err != nil { + if err := kubernetes.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitPodReadyTimeout); err != nil { return fmt.Errorf("wait karmada search pod status ready timeout: %v", err) } diff --git a/pkg/karmadactl/cmdinit/kubernetes/check.go b/pkg/karmadactl/cmdinit/kubernetes/check.go index fad18b01ba0e..fb9a4617b17e 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/check.go +++ b/pkg/karmadactl/cmdinit/kubernetes/check.go @@ -2,106 +2,83 @@ package kubernetes import ( "context" - "fmt" "time" - corev1 "k8s.io/api/core/v1" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) -func podStatus(pod *corev1.Pod) string { - for _, value := range pod.Status.ContainerStatuses { - if pod.Status.Phase == corev1.PodRunning { - if value.State.Waiting != nil { - return value.State.Waiting.Reason - } - if value.State.Waiting == nil { - return string(corev1.PodRunning) - } - return "Error" - } - if pod.ObjectMeta.DeletionTimestamp != nil { - return "Terminating" - } - } - return pod.Status.ContainerStatuses[0].State.Waiting.Reason -} - -func isPodReady(c kubernetes.Interface, n, p string) wait.ConditionFunc { - return func() (done bool, err error) { - pod, err := c.CoreV1().Pods(n).Get(context.TODO(), p, metav1.GetOptions{}) +// WaitForStatefulsetRollout wait statefulset rollout +func WaitForStatefulsetRollout(c kubernetes.Interface, sts *appsv1.StatefulSet, timeout int) error { + var lastErr error + if err := wait.Poll(time.Second, time.Duration(timeout)*time.Second, func() (bool, error) { + s, err := c.AppsV1().StatefulSets(sts.GetNamespace()).Get(context.TODO(), sts.GetName(), metav1.GetOptions{}) if err != nil { - return false, err + lastErr = err + klog.V(4).ErrorS(err, "WaitForStatefulsetRollout: failed to get StatefulSet") + return false, nil } - - if pod.Status.Phase == corev1.PodPending && len(pod.Status.ContainerStatuses) == 0 { - klog.Warningf("Pod: %s not ready. status: %v", pod.Name, corev1.PodPending) + if s.Generation > s.Status.ObservedGeneration { + lastErr = errors.Errorf("expected generation %d, observed generation: %d", + s.Generation, s.Status.ObservedGeneration) return false, nil } - - for _, v := range pod.Status.Conditions { - switch v.Type { - case corev1.PodReady: - if v.Status == corev1.ConditionTrue { - klog.Infof("pod: %s is ready. status: %v", pod.Name, podStatus(pod)) - return true, nil - } - klog.Warningf("Pod: %s not ready. status: %v", pod.Name, podStatus(pod)) - return false, nil - default: - continue - } + if s.Status.UpdatedReplicas != s.Status.Replicas { + lastErr = errors.Errorf("expected %d replicas, got %d updated replicas", + s.Status.Replicas, s.Status.UpdatedReplicas) + return false, nil } - return false, err - } -} - -// waitPodReady Poll up to timeout seconds for pod to enter running state. -// Returns an error if the pod never enters the running state. -func waitPodReady(c kubernetes.Interface, namespaces, podName string, timeout time.Duration) error { - return wait.PollImmediate(time.Second, timeout, isPodReady(c, namespaces, podName)) -} - -// WaitPodReady wait pod ready -func WaitPodReady(c kubernetes.Interface, namespace, selector string, timeout int) error { - // Wait 3 second - time.Sleep(3 * time.Second) - pods, err := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector}) - if err != nil { - return err - } - - if len(pods.Items) == 0 { - return fmt.Errorf("no pods in %s with selector %s", namespace, selector) - } - - for _, pod := range pods.Items { - if err = waitPodReady(c, namespace, pod.Name, time.Duration(timeout)*time.Second); err != nil { - return err + if s.Status.ReadyReplicas != s.Status.Replicas { + lastErr = errors.Errorf("expected %d replicas, got %d ready replicas", + s.Status.Replicas, s.Status.ReadyReplicas) + return false, nil } + return true, nil + }); err != nil { + if err == wait.ErrWaitTimeout && lastErr != nil { + err = lastErr + } + return errors.Wrapf(err, "waiting for StatefulsetRollout of %s/%s", sts.GetNamespace(), sts.GetName()) } - return nil } -// WaitEtcdReplicasetInDesired Wait Etcd Ready -func WaitEtcdReplicasetInDesired(replicas int32, c kubernetes.Interface, namespace, selector string, timeout int) error { - if err := wait.PollImmediate(time.Second, time.Duration(timeout)*time.Second, func() (done bool, err error) { - pods, e := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector}) - if e != nil { +// WaitForDeploymentRollout wait deployment rollout +func WaitForDeploymentRollout(c kubernetes.Interface, dep *appsv1.Deployment, timeout int) error { + var lastErr error + if err := wait.Poll(time.Second, time.Duration(timeout)*time.Second, func() (bool, error) { + d, err := c.AppsV1().Deployments(dep.GetNamespace()).Get(context.TODO(), dep.GetName(), metav1.GetOptions{}) + if err != nil { + lastErr = err + klog.V(4).ErrorS(err, "WaitForDeploymentRollout: failed to get Deployment") + return false, nil + } + if d.Generation > d.Status.ObservedGeneration { + lastErr = errors.Errorf("current generation %d, observed generation %d", + d.Generation, d.Status.ObservedGeneration) return false, nil } - if int32(len(pods.Items)) == replicas { - klog.Infof("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items)) - return true, nil + if d.Status.UpdatedReplicas != d.Status.Replicas { + lastErr = errors.Errorf("the number of pods targeted by the deployment (%d pods) is different "+ + "from the number of pods targeted by the deployment that have the desired template spec (%d pods)", + d.Status.Replicas, d.Status.UpdatedReplicas) + return false, nil } - klog.Warningf("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items)) - return false, nil + if d.Status.UnavailableReplicas != 0 { + lastErr = errors.Errorf("got %d unavailable replicas", + d.Status.UnavailableReplicas) + return false, nil + } + return true, nil }); err != nil { - return err + if err == wait.ErrWaitTimeout && lastErr != nil { + err = lastErr + } + return errors.Wrapf(err, "waiting for DeploymentRollout of %s/%s", dep.GetNamespace(), dep.GetName()) } return nil } diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index bae675ca33d5..1c636e9fc94f 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -351,24 +351,23 @@ func (i *CommandInitOption) initKarmadaAPIServer() error { return err } klog.Info("Create etcd StatefulSets") - if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), i.makeETCDStatefulSet(), metav1.CreateOptions{}); err != nil { + etcdStatefulSet := i.makeETCDStatefulSet() + if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), etcdStatefulSet, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitEtcdReplicasetInDesired(i.EtcdReplicas, i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil { + if err := WaitForStatefulsetRollout(i.KubeClientSet, etcdStatefulSet, 30); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil { - klog.Warning(err) - } - klog.Info("Create karmada ApiServer Deployment") if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil { return err } - if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAPIServerDeployment(), metav1.CreateOptions{}); err != nil { + + karmadaAPIServerDeployment := i.makeKarmadaAPIServerDeployment() + if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAPIServerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(apiServerLabels), 120); err != nil { + if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaAPIServerDeployment, 120); err != nil { return err } @@ -378,10 +377,11 @@ func (i *CommandInitOption) initKarmadaAPIServer() error { if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaAggregatedAPIServerService()); err != nil { klog.Exitln(err) } + karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment() if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(aggregatedAPIServerLabels), 30); err != nil { + if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, 30); err != nil { klog.Warning(err) } return nil @@ -398,30 +398,33 @@ func (i *CommandInitOption) initKarmadaComponent() error { if err := util.CreateOrUpdateService(i.KubeClientSet, i.kubeControllerManagerService()); err != nil { klog.Exitln(err) } - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaKubeControllerManagerDeployment(), metav1.CreateOptions{}); err != nil { + karmadaKubeControllerManagerDeployment := i.makeKarmadaKubeControllerManagerDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaKubeControllerManagerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(kubeControllerManagerLabels), waitPodReadyTimeout); err != nil { + if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaKubeControllerManagerDeployment, waitPodReadyTimeout); err != nil { klog.Warning(err) } // Create karmada-scheduler // https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-scheduler.yaml klog.Info("Create karmada scheduler Deployment") - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaSchedulerDeployment(), metav1.CreateOptions{}); err != nil { + karmadaSchedulerDeployment := i.makeKarmadaSchedulerDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaSchedulerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(schedulerLabels), waitPodReadyTimeout); err != nil { + if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaSchedulerDeployment, waitPodReadyTimeout); err != nil { klog.Warning(err) } // Create karmada-controller-manager // https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-controller-manager.yaml klog.Info("Create karmada controller manager Deployment") - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaControllerManagerDeployment(), metav1.CreateOptions{}); err != nil { + karmadaControllerManagerDeployment := i.makeKarmadaControllerManagerDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaControllerManagerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(controllerManagerLabels), waitPodReadyTimeout); err != nil { + if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaControllerManagerDeployment, waitPodReadyTimeout); err != nil { klog.Warning(err) } @@ -431,10 +434,11 @@ func (i *CommandInitOption) initKarmadaComponent() error { if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaWebhookService()); err != nil { klog.Exitln(err) } - if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaWebhookDeployment(), metav1.CreateOptions{}); err != nil { + karmadaWebhookDeployment := i.makeKarmadaWebhookDeployment() + if _, err := deploymentClient.Create(context.TODO(), karmadaWebhookDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } - if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(webhookLabels), waitPodReadyTimeout); err != nil { + if err := WaitForDeploymentRollout(i.KubeClientSet, karmadaWebhookDeployment, waitPodReadyTimeout); err != nil { klog.Warning(err) } return nil diff --git a/pkg/karmadactl/register/register.go b/pkg/karmadactl/register/register.go index 8fd3063f1c89..077d24d90b14 100644 --- a/pkg/karmadactl/register/register.go +++ b/pkg/karmadactl/register/register.go @@ -35,7 +35,6 @@ import ( "github.com/karmada-io/karmada/pkg/apis/cluster/validation" check "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" - "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -349,11 +348,12 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { // create karmada-agent Deployment in the member cluster fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment") - if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), o.makeKarmadaAgentDeployment(), metav1.CreateOptions{}); err != nil { + KarmadaAgentDeployment := o.makeKarmadaAgentDeployment() + if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { return err } - if err := check.WaitPodReady(o.memberClusterClient, o.Namespace, utils.MapToString(karmadaAgentLabels), int(o.Timeout)); err != nil { + if err := check.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err }