diff --git a/pkg/descheduler/core/helper.go b/pkg/descheduler/core/helper.go index c5e257534a78..f2f31a75343c 100644 --- a/pkg/descheduler/core/helper.go +++ b/pkg/descheduler/core/helper.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "math" + "sort" "time" "github.com/kr/pretty" @@ -62,24 +63,39 @@ func NewSchedulingResultHelper(binding *workv1alpha2.ResourceBinding) *Schedulin func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshold time.Duration) { reference := &h.Spec.Resource undesiredClusters, undesiredClusterNames := h.GetUndesiredClusters() + // Set the boundary. for i := range undesiredClusters { undesiredClusters[i].Unschedulable = math.MaxInt32 } - // Get the minimum value of MaxAvailableReplicas in terms of all estimators. + + // get estimators and sort it by priority. estimators := estimatorclient.GetUnschedulableReplicaEstimators() + sort.Slice(estimators, func(i, j int) bool { + return estimators[i].GetPriority() > estimators[j].GetPriority() + }) + + priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32) ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", reference.Kind, reference.Namespace, reference.Name)) + for _, estimator := range estimators { + // if higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator. + if estimator.GetPriority() < priorityOfAvailableEstimator && clustersFullyEstimated(undesiredClusters) { + break + } res, err := estimator.GetUnschedulableReplicas(ctx, undesiredClusterNames, reference, unschedulableThreshold) if err != nil { klog.Errorf("Max cluster unschedulable replicas error: %v", err) continue } + priorityOfAvailableEstimator = estimator.GetPriority() for i := range res { + // if an estimator can only give a part of result of member clusters, same priority estimator will be called. if res[i].Replicas == estimatorclient.UnauthenticReplica { continue } + // if two estimators are of the same priority, call both and choose the minimum value of each estimated result. if undesiredClusters[i].ClusterName == res[i].Name && undesiredClusters[i].Unschedulable > res[i].Replicas { undesiredClusters[i].Unschedulable = res[i].Replicas } @@ -95,6 +111,16 @@ func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshol klog.V(4).Infof("Target undesired cluster of unschedulable replica result: %s", pretty.Sprint(undesiredClusters)) } +// clustersFullyEstimated whether estimators has gaven a full result of member clusters +func clustersFullyEstimated(undesiredClusters []*TargetClusterWrapper) bool { + for i := range undesiredClusters { + if undesiredClusters[i].Unschedulable == math.MaxInt32 { + return false + } + } + return true +} + // GetUndesiredClusters returns the cluster which of ready replicas are not reach the ready ones. func (h *SchedulingResultHelper) GetUndesiredClusters() ([]*TargetClusterWrapper, []string) { var clusters []*TargetClusterWrapper diff --git a/pkg/descheduler/descheduler_test.go b/pkg/descheduler/descheduler_test.go index 0b70bd2fb98e..d4f31e7e3af9 100644 --- a/pkg/descheduler/descheduler_test.go +++ b/pkg/descheduler/descheduler_test.go @@ -487,6 +487,7 @@ func TestDescheduler_worker(t *testing.T) { } schedulerEstimator := estimatorclient.NewSchedulerEstimator(desched.schedulerEstimatorCache, 5*time.Second) estimatorclient.RegisterSchedulerEstimator(schedulerEstimator) + defer estimatorclient.UnRegisterSchedulerEstimator(schedulerEstimator) for _, c := range tt.args.unschedulable { cluster := c diff --git a/pkg/estimator/client/accurate.go b/pkg/estimator/client/accurate.go index e5390fe0c200..68f08f4b7c65 100644 --- a/pkg/estimator/client/accurate.go +++ b/pkg/estimator/client/accurate.go @@ -32,8 +32,24 @@ import ( // RegisterSchedulerEstimator will register a SchedulerEstimator. func RegisterSchedulerEstimator(se *SchedulerEstimator) { - replicaEstimators["scheduler-estimator"] = se - unschedulableReplicaEstimators["scheduler-estimator"] = se + replicaEstimators = append(replicaEstimators, se) + unschedulableReplicaEstimators = append(unschedulableReplicaEstimators, se) +} + +// UnRegisterSchedulerEstimator will unregister a SchedulerEstimator. +func UnRegisterSchedulerEstimator(se *SchedulerEstimator) { + for i := 0; i < len(replicaEstimators); i++ { + if replicaEstimators[i] == se { + replicaEstimators = append(replicaEstimators[:i], replicaEstimators[i+1:]...) + i-- + } + } + for i := 0; i < len(unschedulableReplicaEstimators); i++ { + if unschedulableReplicaEstimators[i] == se { + unschedulableReplicaEstimators = append(unschedulableReplicaEstimators[:i], unschedulableReplicaEstimators[i+1:]...) + i-- + } + } } type getClusterReplicasFunc func(ctx context.Context, cluster string) (int32, error) @@ -67,6 +83,10 @@ func (se *SchedulerEstimator) MaxAvailableReplicas( }) } +func (se *SchedulerEstimator) GetPriority() EstimatorPriority { + return Accurate +} + // GetUnschedulableReplicas gets the unschedulable replicas which belong to a specified workload by calling karmada-scheduler-estimator. func (se *SchedulerEstimator) GetUnschedulableReplicas( parentCtx context.Context, diff --git a/pkg/estimator/client/general.go b/pkg/estimator/client/general.go index 6023c89f528a..9d2e4a305dbf 100644 --- a/pkg/estimator/client/general.go +++ b/pkg/estimator/client/general.go @@ -32,7 +32,7 @@ import ( // GeneralEstimator is the default replica estimator. func init() { - replicaEstimators["general-estimator"] = NewGeneralEstimator() + replicaEstimators = append(replicaEstimators, NewGeneralEstimator()) } // GeneralEstimator is a normal estimator in terms of cluster ResourceSummary. @@ -53,6 +53,10 @@ func (ge *GeneralEstimator) MaxAvailableReplicas(_ context.Context, clusters []* return availableTargetClusters, nil } +func (ge *GeneralEstimator) GetPriority() EstimatorPriority { + return General +} + func (ge *GeneralEstimator) maxAvailableReplicas(cluster *clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) int32 { resourceSummary := cluster.Status.ResourceSummary if resourceSummary == nil { diff --git a/pkg/estimator/client/interface.go b/pkg/estimator/client/interface.go index 2eab32a90ef7..658f45964e1b 100644 --- a/pkg/estimator/client/interface.go +++ b/pkg/estimator/client/interface.go @@ -30,26 +30,38 @@ import ( const UnauthenticReplica = -1 var ( - replicaEstimators = map[string]ReplicaEstimator{} - unschedulableReplicaEstimators = map[string]UnschedulableReplicaEstimator{} + replicaEstimators []ReplicaEstimator + unschedulableReplicaEstimators []UnschedulableReplicaEstimator ) // ReplicaEstimator is an estimator which estimates the maximum replicas that can be applied to the target cluster. type ReplicaEstimator interface { MaxAvailableReplicas(ctx context.Context, clusters []*clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) ([]workv1alpha2.TargetCluster, error) + GetPriority() EstimatorPriority } // UnschedulableReplicaEstimator is an estimator which estimates the unschedulable replicas which belong to a specified workload. type UnschedulableReplicaEstimator interface { GetUnschedulableReplicas(ctx context.Context, clusters []string, reference *workv1alpha2.ObjectReference, unschedulableThreshold time.Duration) ([]workv1alpha2.TargetCluster, error) + GetPriority() EstimatorPriority } // GetReplicaEstimators returns all replica estimators. -func GetReplicaEstimators() map[string]ReplicaEstimator { +func GetReplicaEstimators() []ReplicaEstimator { return replicaEstimators } // GetUnschedulableReplicaEstimators returns all unschedulable replica estimators. -func GetUnschedulableReplicaEstimators() map[string]UnschedulableReplicaEstimator { +func GetUnschedulableReplicaEstimators() []UnschedulableReplicaEstimator { return unschedulableReplicaEstimators } + +// EstimatorPriority +// 1. If two estimators are of the same priority, call both and choose the minimum value of each estimated result. +// 2. If higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator. +type EstimatorPriority int32 + +const ( + General EstimatorPriority = 10 + Accurate EstimatorPriority = 20 +) diff --git a/pkg/scheduler/core/util.go b/pkg/scheduler/core/util.go index 80dcfca0d153..0e3214b09f1f 100644 --- a/pkg/scheduler/core/util.go +++ b/pkg/scheduler/core/util.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "math" + "sort" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -68,20 +69,33 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha return availableTargetClusters } - // Get the minimum value of MaxAvailableReplicas in terms of all estimators. + // get estimators and sort it by priority. estimators := estimatorclient.GetReplicaEstimators() + sort.Slice(estimators, func(i, j int) bool { + return estimators[i].GetPriority() > estimators[j].GetPriority() + }) + + priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32) ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", spec.Resource.Kind, spec.Resource.Namespace, spec.Resource.Name)) + for _, estimator := range estimators { + // if higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator. + if estimator.GetPriority() < priorityOfAvailableEstimator && ClustersFullyEstimated(availableTargetClusters) { + break + } res, err := estimator.MaxAvailableReplicas(ctx, clusters, spec.ReplicaRequirements) if err != nil { klog.Errorf("Max cluster available replicas error: %v", err) continue } + priorityOfAvailableEstimator = estimator.GetPriority() for i := range res { + // if an estimator can only give a part of result of member clusters, same priority estimator will be called. if res[i].Replicas == estimatorclient.UnauthenticReplica { continue } + // if two estimators are of the same priority, call both and choose the minimum value of each estimated result. if availableTargetClusters[i].Name == res[i].Name && availableTargetClusters[i].Replicas > res[i].Replicas { availableTargetClusters[i].Replicas = res[i].Replicas } @@ -100,6 +114,16 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha return availableTargetClusters } +// ClustersFullyEstimated whether estimators has gaven a full result of member clusters +func ClustersFullyEstimated(availableTargetClusters []workv1alpha2.TargetCluster) bool { + for i := range availableTargetClusters { + if availableTargetClusters[i].Replicas == math.MaxInt32 { + return false + } + } + return true +} + // attachZeroReplicasCluster attach cluster in clusters into targetCluster // The purpose is to avoid workload not appeared in rb's spec.clusters field func attachZeroReplicasCluster(clusters []*clusterv1alpha1.Cluster, targetClusters []workv1alpha2.TargetCluster) []workv1alpha2.TargetCluster {