diff --git a/pkg/descheduler/core/helper.go b/pkg/descheduler/core/helper.go index c5e257534a78..f2f31a75343c 100644 --- a/pkg/descheduler/core/helper.go +++ b/pkg/descheduler/core/helper.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "math" + "sort" "time" "github.com/kr/pretty" @@ -62,24 +63,39 @@ func NewSchedulingResultHelper(binding *workv1alpha2.ResourceBinding) *Schedulin func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshold time.Duration) { reference := &h.Spec.Resource undesiredClusters, undesiredClusterNames := h.GetUndesiredClusters() + // Set the boundary. for i := range undesiredClusters { undesiredClusters[i].Unschedulable = math.MaxInt32 } - // Get the minimum value of MaxAvailableReplicas in terms of all estimators. + + // get estimators and sort it by priority. estimators := estimatorclient.GetUnschedulableReplicaEstimators() + sort.Slice(estimators, func(i, j int) bool { + return estimators[i].GetPriority() > estimators[j].GetPriority() + }) + + priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32) ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", reference.Kind, reference.Namespace, reference.Name)) + for _, estimator := range estimators { + // if higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator. + if estimator.GetPriority() < priorityOfAvailableEstimator && clustersFullyEstimated(undesiredClusters) { + break + } res, err := estimator.GetUnschedulableReplicas(ctx, undesiredClusterNames, reference, unschedulableThreshold) if err != nil { klog.Errorf("Max cluster unschedulable replicas error: %v", err) continue } + priorityOfAvailableEstimator = estimator.GetPriority() for i := range res { + // if an estimator can only give a part of result of member clusters, same priority estimator will be called. if res[i].Replicas == estimatorclient.UnauthenticReplica { continue } + // if two estimators are of the same priority, call both and choose the minimum value of each estimated result. if undesiredClusters[i].ClusterName == res[i].Name && undesiredClusters[i].Unschedulable > res[i].Replicas { undesiredClusters[i].Unschedulable = res[i].Replicas } @@ -95,6 +111,16 @@ func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshol klog.V(4).Infof("Target undesired cluster of unschedulable replica result: %s", pretty.Sprint(undesiredClusters)) } +// clustersFullyEstimated whether estimators has gaven a full result of member clusters +func clustersFullyEstimated(undesiredClusters []*TargetClusterWrapper) bool { + for i := range undesiredClusters { + if undesiredClusters[i].Unschedulable == math.MaxInt32 { + return false + } + } + return true +} + // GetUndesiredClusters returns the cluster which of ready replicas are not reach the ready ones. func (h *SchedulingResultHelper) GetUndesiredClusters() ([]*TargetClusterWrapper, []string) { var clusters []*TargetClusterWrapper diff --git a/pkg/estimator/client/accurate.go b/pkg/estimator/client/accurate.go index e5390fe0c200..0e7549a6932d 100644 --- a/pkg/estimator/client/accurate.go +++ b/pkg/estimator/client/accurate.go @@ -32,8 +32,8 @@ import ( // RegisterSchedulerEstimator will register a SchedulerEstimator. func RegisterSchedulerEstimator(se *SchedulerEstimator) { - replicaEstimators["scheduler-estimator"] = se - unschedulableReplicaEstimators["scheduler-estimator"] = se + replicaEstimators = append(replicaEstimators, se) + unschedulableReplicaEstimators = append(unschedulableReplicaEstimators, se) } type getClusterReplicasFunc func(ctx context.Context, cluster string) (int32, error) @@ -67,6 +67,10 @@ func (se *SchedulerEstimator) MaxAvailableReplicas( }) } +func (se *SchedulerEstimator) GetPriority() EstimatorPriority { + return Accurate +} + // GetUnschedulableReplicas gets the unschedulable replicas which belong to a specified workload by calling karmada-scheduler-estimator. func (se *SchedulerEstimator) GetUnschedulableReplicas( parentCtx context.Context, diff --git a/pkg/estimator/client/general.go b/pkg/estimator/client/general.go index 6023c89f528a..9d2e4a305dbf 100644 --- a/pkg/estimator/client/general.go +++ b/pkg/estimator/client/general.go @@ -32,7 +32,7 @@ import ( // GeneralEstimator is the default replica estimator. func init() { - replicaEstimators["general-estimator"] = NewGeneralEstimator() + replicaEstimators = append(replicaEstimators, NewGeneralEstimator()) } // GeneralEstimator is a normal estimator in terms of cluster ResourceSummary. @@ -53,6 +53,10 @@ func (ge *GeneralEstimator) MaxAvailableReplicas(_ context.Context, clusters []* return availableTargetClusters, nil } +func (ge *GeneralEstimator) GetPriority() EstimatorPriority { + return General +} + func (ge *GeneralEstimator) maxAvailableReplicas(cluster *clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) int32 { resourceSummary := cluster.Status.ResourceSummary if resourceSummary == nil { diff --git a/pkg/estimator/client/interface.go b/pkg/estimator/client/interface.go index 2eab32a90ef7..658f45964e1b 100644 --- a/pkg/estimator/client/interface.go +++ b/pkg/estimator/client/interface.go @@ -30,26 +30,38 @@ import ( const UnauthenticReplica = -1 var ( - replicaEstimators = map[string]ReplicaEstimator{} - unschedulableReplicaEstimators = map[string]UnschedulableReplicaEstimator{} + replicaEstimators []ReplicaEstimator + unschedulableReplicaEstimators []UnschedulableReplicaEstimator ) // ReplicaEstimator is an estimator which estimates the maximum replicas that can be applied to the target cluster. type ReplicaEstimator interface { MaxAvailableReplicas(ctx context.Context, clusters []*clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) ([]workv1alpha2.TargetCluster, error) + GetPriority() EstimatorPriority } // UnschedulableReplicaEstimator is an estimator which estimates the unschedulable replicas which belong to a specified workload. type UnschedulableReplicaEstimator interface { GetUnschedulableReplicas(ctx context.Context, clusters []string, reference *workv1alpha2.ObjectReference, unschedulableThreshold time.Duration) ([]workv1alpha2.TargetCluster, error) + GetPriority() EstimatorPriority } // GetReplicaEstimators returns all replica estimators. -func GetReplicaEstimators() map[string]ReplicaEstimator { +func GetReplicaEstimators() []ReplicaEstimator { return replicaEstimators } // GetUnschedulableReplicaEstimators returns all unschedulable replica estimators. -func GetUnschedulableReplicaEstimators() map[string]UnschedulableReplicaEstimator { +func GetUnschedulableReplicaEstimators() []UnschedulableReplicaEstimator { return unschedulableReplicaEstimators } + +// EstimatorPriority +// 1. If two estimators are of the same priority, call both and choose the minimum value of each estimated result. +// 2. If higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator. +type EstimatorPriority int32 + +const ( + General EstimatorPriority = 10 + Accurate EstimatorPriority = 20 +) diff --git a/pkg/scheduler/core/util.go b/pkg/scheduler/core/util.go index 80dcfca0d153..0e3214b09f1f 100644 --- a/pkg/scheduler/core/util.go +++ b/pkg/scheduler/core/util.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "math" + "sort" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -68,20 +69,33 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha return availableTargetClusters } - // Get the minimum value of MaxAvailableReplicas in terms of all estimators. + // get estimators and sort it by priority. estimators := estimatorclient.GetReplicaEstimators() + sort.Slice(estimators, func(i, j int) bool { + return estimators[i].GetPriority() > estimators[j].GetPriority() + }) + + priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32) ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", spec.Resource.Kind, spec.Resource.Namespace, spec.Resource.Name)) + for _, estimator := range estimators { + // if higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator. + if estimator.GetPriority() < priorityOfAvailableEstimator && ClustersFullyEstimated(availableTargetClusters) { + break + } res, err := estimator.MaxAvailableReplicas(ctx, clusters, spec.ReplicaRequirements) if err != nil { klog.Errorf("Max cluster available replicas error: %v", err) continue } + priorityOfAvailableEstimator = estimator.GetPriority() for i := range res { + // if an estimator can only give a part of result of member clusters, same priority estimator will be called. if res[i].Replicas == estimatorclient.UnauthenticReplica { continue } + // if two estimators are of the same priority, call both and choose the minimum value of each estimated result. if availableTargetClusters[i].Name == res[i].Name && availableTargetClusters[i].Replicas > res[i].Replicas { availableTargetClusters[i].Replicas = res[i].Replicas } @@ -100,6 +114,16 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha return availableTargetClusters } +// ClustersFullyEstimated whether estimators has gaven a full result of member clusters +func ClustersFullyEstimated(availableTargetClusters []workv1alpha2.TargetCluster) bool { + for i := range availableTargetClusters { + if availableTargetClusters[i].Replicas == math.MaxInt32 { + return false + } + } + return true +} + // attachZeroReplicasCluster attach cluster in clusters into targetCluster // The purpose is to avoid workload not appeared in rb's spec.clusters field func attachZeroReplicasCluster(clusters []*clusterv1alpha1.Cluster, targetClusters []workv1alpha2.TargetCluster) []workv1alpha2.TargetCluster {