Merge pull request karmada-io#3010 from yanfeng1992/fixbug-search-not…

…Ready-cluster

search filter out not ready cluster

pkg/karmadactl/cmdinit/kubernetes/node.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
@@ -26,43 +27,32 @@ func (i *CommandInitOption) getKarmadaAPIServerIP() error {
 		return nil
 	}
 
-	nodeClient := i.KubeClientSet.CoreV1().Nodes()
-	masterNodes, err := nodeClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"})
+	nodes, err := i.KubeClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return err
 	}
-	if len(masterNodes.Items) == 0 {
-		masterNodes, err = nodeClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/control-plane"})
-		if err != nil {
-			return err
-		}
-	}
 
-	if len(masterNodes.Items) == 0 {
-		klog.Warning("The kubernetes cluster does not have a Master role.")
-	} else {
-		for _, v := range masterNodes.Items {
-			i.KarmadaAPIServerIP = append(i.KarmadaAPIServerIP, utils.StringToNetIP(v.Status.Addresses[0].Address))
+	TempKarmadaAPIServerIP := make([]net.IP, 0)
+	for ni := range nodes.Items {
+		if isMasterNode(&nodes.Items[ni]) {
+			i.KarmadaAPIServerIP = append(i.KarmadaAPIServerIP, utils.StringToNetIP(nodes.Items[ni].Status.Addresses[0].Address))
 		}
-		return nil
-	}
-
-	klog.Info("Randomly select 3 Node IPs in the kubernetes cluster.")
-	nodes, err := nodeClient.List(context.TODO(), metav1.ListOptions{})
-	if err != nil {
-		return err
-	}
 
-	for number := 0; number < 3; number++ {
-		if number >= len(nodes.Items) {
-			break
+		if ni < 3 {
+			TempKarmadaAPIServerIP = append(i.KarmadaAPIServerIP, utils.StringToNetIP(nodes.Items[ni].Status.Addresses[0].Address))
 		}
-		i.KarmadaAPIServerIP = append(i.KarmadaAPIServerIP, utils.StringToNetIP(nodes.Items[number].Status.Addresses[0].Address))
 	}
 
 	if len(i.KarmadaAPIServerIP) == 0 {
+		if len(TempKarmadaAPIServerIP) != 0 {
+			klog.Warning("The kubernetes cluster does not have a Master role.")
+			klog.Info("Randomly select 3 Node IPs in the kubernetes cluster.")
+			i.KarmadaAPIServerIP = TempKarmadaAPIServerIP
+			return nil
+		}
 		return fmt.Errorf("karmada apiserver ip is empty")
 	}
+
 	return nil
 }
 
@@ -142,3 +132,16 @@ func (i *CommandInitOption) isNodeExist(labels string) bool {
 	klog.Infof("Find the node [ %s ] by label %s", node.Items[0].Name, labels)
 	return true
 }
+
+// see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md#urgent-upgrade-notes
+func isMasterNode(node *corev1.Node) bool {
+	if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
+		return ok
+	}
+
+	if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
+		return ok
+	}
+
+	return false
+}

pkg/search/controller.go

@@ -171,7 +171,7 @@ func (c *Controller) doCacheCluster(cluster string) error {
 		return nil
 	}
 
-	// STEP1: stop informer manager for the cluster which does not exist anymore.
+	// STEP1: stop informer manager for the cluster which does not exist anymore or is not ready.
 	cls, err := c.clusterLister.Get(cluster)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
@@ -188,6 +188,12 @@ func (c *Controller) doCacheCluster(cluster string) error {
 		return nil
 	}
 
+	if !util.IsClusterReady(&cls.Status) {
+		klog.Warningf("cluster %s is notReady try to stop this cluster informer", cluster)
+		c.InformerManager.Stop(cluster)
+		return nil
+	}
+
 	// STEP2: added/updated cluster, builds an informer manager for a specific cluster.
 	if !c.InformerManager.IsManagerExist(cluster) {
 		klog.Info("Try to build informer manager for cluster ", cluster)

pkg/search/proxy/controller.go

@@ -182,6 +182,12 @@ func (ctl *Controller) reconcile(util.QueueKey) error {
 			if !util.ClusterMatches(cluster, registry.Spec.TargetCluster) {
 				continue
 			}
+
+			if !util.IsClusterReady(&cluster.Status) {
+				klog.Warningf("cluster %s is notReady", cluster.Name)
+				continue
+			}
+
 			if _, exist := resourcesByClusters[cluster.Name]; !exist {
 				resourcesByClusters[cluster.Name] = make(map[schema.GroupVersionResource]struct{})
 			}

pkg/search/proxy/controller_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/karmada-io/karmada/pkg/search/proxy/framework"
 	pluginruntime "github.com/karmada-io/karmada/pkg/search/proxy/framework/runtime"
 	proxytest "github.com/karmada-io/karmada/pkg/search/proxy/testing"
+	"github.com/karmada-io/karmada/pkg/util"
 )
 
 func TestController(t *testing.T) {
@@ -493,5 +494,8 @@ func TestController_Connect_Error(t *testing.T) {
 func newCluster(name string) *clusterv1alpha1.Cluster {
 	c := &clusterv1alpha1.Cluster{}
 	c.Name = name
+	conditions := make([]metav1.Condition, 0, 1)
+	conditions = append(conditions, util.NewCondition(clusterv1alpha1.ClusterConditionReady, "", "", metav1.ConditionTrue))
+	c.Status.Conditions = conditions
 	return c
 }