From f46a56477d8a6c9dd7f5ee0f1fcfc68486ae3766 Mon Sep 17 00:00:00 2001 From: chaosi-zju Date: Mon, 11 Dec 2023 16:01:12 +0800 Subject: [PATCH] adjust the Priority of General and Accurate Estimator of Karmada Scheduler Signed-off-by: chaosi-zju --- pkg/descheduler/core/helper.go | 18 +++++++++++++++++- pkg/estimator/client/accurate.go | 8 ++++++-- pkg/estimator/client/general.go | 6 +++++- pkg/estimator/client/interface.go | 22 ++++++++++++++++++---- pkg/scheduler/core/util.go | 16 +++++++++++++++- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/pkg/descheduler/core/helper.go b/pkg/descheduler/core/helper.go index c5e257534a78..72d0531f8348 100644 --- a/pkg/descheduler/core/helper.go +++ b/pkg/descheduler/core/helper.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "math" + "sort" "time" "github.com/kr/pretty" @@ -62,24 +63,39 @@ func NewSchedulingResultHelper(binding *workv1alpha2.ResourceBinding) *Schedulin func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshold time.Duration) { reference := &h.Spec.Resource undesiredClusters, undesiredClusterNames := h.GetUndesiredClusters() + // Set the boundary. for i := range undesiredClusters { undesiredClusters[i].Unschedulable = math.MaxInt32 } - // Get the minimum value of MaxAvailableReplicas in terms of all estimators. + + // get estimators and sort it by priority. estimators := estimatorclient.GetUnschedulableReplicaEstimators() + sort.Slice(estimators, func(i, j int) bool { + return estimators[i].GetPriority() > estimators[j].GetPriority() + }) + + priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32) ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", reference.Kind, reference.Namespace, reference.Name)) + for _, estimator := range estimators { + // if higher-priority estimator has given the result, no longer to call lower-priority estimator. + if estimator.GetPriority() < priorityOfAvailableEstimator { + break + } res, err := estimator.GetUnschedulableReplicas(ctx, undesiredClusterNames, reference, unschedulableThreshold) if err != nil { klog.Errorf("Max cluster unschedulable replicas error: %v", err) continue } + priorityOfAvailableEstimator = estimator.GetPriority() for i := range res { + // if an estimator can only give a part of result of member clusters, same priority estimator will be called, but low-priority estimator not. if res[i].Replicas == estimatorclient.UnauthenticReplica { continue } + // if two estimators are of the same priority, call both and choose the minimum value of each estimated result. if undesiredClusters[i].ClusterName == res[i].Name && undesiredClusters[i].Unschedulable > res[i].Replicas { undesiredClusters[i].Unschedulable = res[i].Replicas } diff --git a/pkg/estimator/client/accurate.go b/pkg/estimator/client/accurate.go index e5390fe0c200..0e7549a6932d 100644 --- a/pkg/estimator/client/accurate.go +++ b/pkg/estimator/client/accurate.go @@ -32,8 +32,8 @@ import ( // RegisterSchedulerEstimator will register a SchedulerEstimator. func RegisterSchedulerEstimator(se *SchedulerEstimator) { - replicaEstimators["scheduler-estimator"] = se - unschedulableReplicaEstimators["scheduler-estimator"] = se + replicaEstimators = append(replicaEstimators, se) + unschedulableReplicaEstimators = append(unschedulableReplicaEstimators, se) } type getClusterReplicasFunc func(ctx context.Context, cluster string) (int32, error) @@ -67,6 +67,10 @@ func (se *SchedulerEstimator) MaxAvailableReplicas( }) } +func (se *SchedulerEstimator) GetPriority() EstimatorPriority { + return Accurate +} + // GetUnschedulableReplicas gets the unschedulable replicas which belong to a specified workload by calling karmada-scheduler-estimator. func (se *SchedulerEstimator) GetUnschedulableReplicas( parentCtx context.Context, diff --git a/pkg/estimator/client/general.go b/pkg/estimator/client/general.go index 6023c89f528a..9d2e4a305dbf 100644 --- a/pkg/estimator/client/general.go +++ b/pkg/estimator/client/general.go @@ -32,7 +32,7 @@ import ( // GeneralEstimator is the default replica estimator. func init() { - replicaEstimators["general-estimator"] = NewGeneralEstimator() + replicaEstimators = append(replicaEstimators, NewGeneralEstimator()) } // GeneralEstimator is a normal estimator in terms of cluster ResourceSummary. @@ -53,6 +53,10 @@ func (ge *GeneralEstimator) MaxAvailableReplicas(_ context.Context, clusters []* return availableTargetClusters, nil } +func (ge *GeneralEstimator) GetPriority() EstimatorPriority { + return General +} + func (ge *GeneralEstimator) maxAvailableReplicas(cluster *clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) int32 { resourceSummary := cluster.Status.ResourceSummary if resourceSummary == nil { diff --git a/pkg/estimator/client/interface.go b/pkg/estimator/client/interface.go index 2eab32a90ef7..c466a9c5c580 100644 --- a/pkg/estimator/client/interface.go +++ b/pkg/estimator/client/interface.go @@ -30,26 +30,40 @@ import ( const UnauthenticReplica = -1 var ( - replicaEstimators = map[string]ReplicaEstimator{} - unschedulableReplicaEstimators = map[string]UnschedulableReplicaEstimator{} + replicaEstimators []ReplicaEstimator + unschedulableReplicaEstimators []UnschedulableReplicaEstimator ) // ReplicaEstimator is an estimator which estimates the maximum replicas that can be applied to the target cluster. type ReplicaEstimator interface { MaxAvailableReplicas(ctx context.Context, clusters []*clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) ([]workv1alpha2.TargetCluster, error) + GetPriority() EstimatorPriority } // UnschedulableReplicaEstimator is an estimator which estimates the unschedulable replicas which belong to a specified workload. type UnschedulableReplicaEstimator interface { GetUnschedulableReplicas(ctx context.Context, clusters []string, reference *workv1alpha2.ObjectReference, unschedulableThreshold time.Duration) ([]workv1alpha2.TargetCluster, error) + GetPriority() EstimatorPriority } // GetReplicaEstimators returns all replica estimators. -func GetReplicaEstimators() map[string]ReplicaEstimator { +func GetReplicaEstimators() []ReplicaEstimator { return replicaEstimators } // GetUnschedulableReplicaEstimators returns all unschedulable replica estimators. -func GetUnschedulableReplicaEstimators() map[string]UnschedulableReplicaEstimator { +func GetUnschedulableReplicaEstimators() []UnschedulableReplicaEstimator { return unschedulableReplicaEstimators } + +// EstimatorPriority +// 1. If higher-priority estimator has given the result, no longer to call lower-priority estimator. +// 2. If two estimators are of the same priority, call both and choose the minimum value of each estimated result. +// 3. If an estimator can only give a part of result of member clusters, same priority estimator will be called, but low-priority estimator not. +type EstimatorPriority int32 + +const ( + Rough EstimatorPriority = 100 + General EstimatorPriority = 200 + Accurate EstimatorPriority = 300 +) diff --git a/pkg/scheduler/core/util.go b/pkg/scheduler/core/util.go index 80dcfca0d153..73f0428021e5 100644 --- a/pkg/scheduler/core/util.go +++ b/pkg/scheduler/core/util.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "math" + "sort" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -68,20 +69,33 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha return availableTargetClusters } - // Get the minimum value of MaxAvailableReplicas in terms of all estimators. + // get estimators and sort it by priority. estimators := estimatorclient.GetReplicaEstimators() + sort.Slice(estimators, func(i, j int) bool { + return estimators[i].GetPriority() > estimators[j].GetPriority() + }) + + priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32) ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", spec.Resource.Kind, spec.Resource.Namespace, spec.Resource.Name)) + for _, estimator := range estimators { + // if higher-priority estimator has given the result, no longer to call lower-priority estimator. + if estimator.GetPriority() < priorityOfAvailableEstimator { + break + } res, err := estimator.MaxAvailableReplicas(ctx, clusters, spec.ReplicaRequirements) if err != nil { klog.Errorf("Max cluster available replicas error: %v", err) continue } + priorityOfAvailableEstimator = estimator.GetPriority() for i := range res { + // if an estimator can only give a part of result of member clusters, same priority estimator will be called, but low-priority estimator not. if res[i].Replicas == estimatorclient.UnauthenticReplica { continue } + // if two estimators are of the same priority, call both and choose the minimum value of each estimated result. if availableTargetClusters[i].Name == res[i].Name && availableTargetClusters[i].Replicas > res[i].Replicas { availableTargetClusters[i].Replicas = res[i].Replicas }