support cel selector

Signed-off-by: Qing Hao <[email protected]>

go.mod

@@ -2,6 +2,10 @@ module open-cluster-management.io/ocm
 
 go 1.22.5
 
+replace open-cluster-management.io/api => github.com/haoqing0110/api v0.0.0-20250103062352-f89d6a235b5d
+
+replace open-cluster-management.io/sdk-go => github.com/haoqing0110/sdk-go v0.0.0-20250103070425-b9c9ca7d8140
+
 require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/evanphx/json-patch v5.9.0+incompatible

go.sum

@@ -158,6 +158,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
+github.com/haoqing0110/api v0.0.0-20250103062352-f89d6a235b5d h1:z4cj688UOV1I19QcZyvsT9wSeZXGcOoAeOfS+z5chRc=
+github.com/haoqing0110/api v0.0.0-20250103062352-f89d6a235b5d/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
+github.com/haoqing0110/sdk-go v0.0.0-20250103070425-b9c9ca7d8140 h1:LfHKzwJBXCkwhJqt+rc6xEmnLcwcef46295GC8fYMCw=
+github.com/haoqing0110/sdk-go v0.0.0-20250103070425-b9c9ca7d8140/go.mod h1:Kn9PLhwuutwK3hRyCUm8XcHEBWQwnnFGJZ8Z6FYoFEE=
 github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -453,10 +457,6 @@ k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY
 k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50 h1:TXRd6OdGjArh6cwlCYOqlIcyx21k81oUIYj4rmHlYx0=
 open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50/go.mod h1:tsBSNs9mGfVQQjXBnjgpiX6r0UM+G3iNfmzQgKhEfw4=
-open-cluster-management.io/api v0.15.1-0.20241120090202-cb7ce98ab874 h1:WgkuYXTbJV7EK+qtiMq3soa21faGUKeTG5w0C8Mn1Ok=
-open-cluster-management.io/api v0.15.1-0.20241120090202-cb7ce98ab874/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
-open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f h1:zeC7QrFNarfK2zY6jGtd+mX+yDrQQmnH/J8A7n5Nh38=
-open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/cluster-inventory-api v0.0.0-20240730014211-ef0154379848 h1:WYPi2PdQyZwZkHG648v2jQl6deyCgyjJ0fkLYgUJ618=

pkg/placement/controllers/scheduling/scheduling_controller.go

@@ -784,15 +784,16 @@ func filterClustersBySelector(
 ) ([]clusterapiv1beta1.ClusterDecision, *framework.Status) {
 	var matched []clusterapiv1beta1.ClusterDecision
 	// create cluster label selector
-	clusterSelector, err := helpers.NewClusterSelector(selector)
+	// cel evaluator is set to nil currently, can enable it when we decide to support CEL in decision group
+	clusterSelector, err := helpers.NewClusterSelector(selector, nil)
 	if err != nil {
 		status := framework.NewStatus("", framework.Misconfigured, err.Error())
 		return matched, status
 	}
 
 	// filter clusters by label selector
 	for _, cluster := range clusters {
-		if ok := clusterSelector.Matches(cluster.Labels, helpers.GetClusterClaims(cluster)); !ok {
+		if ok := clusterSelector.Matches(cluster); !ok {
 			continue
 		}
 		if !clusterNames.Has(cluster.Name) {

pkg/placement/helpers/clusters.go

@@ -6,14 +6,17 @@ import (
 
 	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
 	clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
+	mclcel "open-cluster-management.io/sdk-go/pkg/cel/managedcluster"
 )
 
 type ClusterSelector struct {
 	labelSelector labels.Selector
 	claimSelector labels.Selector
+	celSelector   *clusterapiv1beta1.ClusterCelSelector
+	celEvaluator  *mclcel.ManagedClusterEvaluator
 }
 
-func NewClusterSelector(selector clusterapiv1beta1.ClusterSelector) (*ClusterSelector, error) {
+func NewClusterSelector(selector clusterapiv1beta1.ClusterSelector, celEvaluator *mclcel.ManagedClusterEvaluator) (*ClusterSelector, error) {
 	// build label selector
 	labelSelector, err := convertLabelSelector(&selector.LabelSelector)
 	if err != nil {
@@ -27,18 +30,28 @@ func NewClusterSelector(selector clusterapiv1beta1.ClusterSelector) (*ClusterSel
 	return &ClusterSelector{
 		labelSelector: labelSelector,
 		claimSelector: claimSelector,
+		celSelector:   &selector.CelSelector,
+		celEvaluator:  celEvaluator,
 	}, nil
 }
 
-func (c *ClusterSelector) Matches(clusterlabels, clusterclaims map[string]string) bool {
+func (c *ClusterSelector) Matches(cluster *clusterapiv1.ManagedCluster) bool {
 	// match with label selector
-	if ok := c.labelSelector.Matches(labels.Set(clusterlabels)); !ok {
+	if ok := c.labelSelector.Matches(labels.Set(cluster.Labels)); !ok {
 		return false
 	}
 	// match with claim selector
-	if ok := c.claimSelector.Matches(labels.Set(clusterclaims)); !ok {
+	if ok := c.claimSelector.Matches(labels.Set(GetClusterClaims(cluster))); !ok {
 		return false
 	}
+	// match with cel selector if exists
+	if c.celEvaluator != nil && len(c.celSelector.CelExpressions) > 0 {
+		ok, err := c.celEvaluator.Evaluate(cluster, c.celSelector.CelExpressions)
+		if err != nil || !ok {
+			return false
+		}
+	}
+
 	return true
 }
 

pkg/placement/helpers/clusters_test.go

@@ -15,8 +15,7 @@ func TestMatches(t *testing.T) {
 	cases := []struct {
 		name            string
 		clusterselector clusterapiv1beta1.ClusterSelector
-		clusterlabels   map[string]string
-		clusterclaims   map[string]string
+		cluster         *clusterapiv1.ManagedCluster
 		expectedMatch   bool
 	}{
 		{
@@ -28,8 +27,7 @@ func TestMatches(t *testing.T) {
 					},
 				},
 			},
-			clusterlabels: map[string]string{"cloud": "Amazon"},
-			clusterclaims: map[string]string{},
+			cluster:       testinghelpers.NewManagedCluster("test").WithLabel("cloud", "Amazon").Build(),
 			expectedMatch: true,
 		},
 		{
@@ -41,8 +39,7 @@ func TestMatches(t *testing.T) {
 					},
 				},
 			},
-			clusterlabels: map[string]string{"cloud": "Google"},
-			clusterclaims: map[string]string{},
+			cluster:       testinghelpers.NewManagedCluster("test").WithLabel("cloud", "Google").Build(),
 			expectedMatch: false,
 		},
 		{
@@ -58,8 +55,7 @@ func TestMatches(t *testing.T) {
 					},
 				},
 			},
-			clusterlabels: map[string]string{},
-			clusterclaims: map[string]string{"cloud": "Amazon"},
+			cluster:       testinghelpers.NewManagedCluster("test").WithClaim("cloud", "Amazon").Build(),
 			expectedMatch: true,
 		},
 		{
@@ -75,8 +71,7 @@ func TestMatches(t *testing.T) {
 					},
 				},
 			},
-			clusterlabels: map[string]string{},
-			clusterclaims: map[string]string{"cloud": "Google"},
+			cluster:       testinghelpers.NewManagedCluster("test").WithClaim("cloud", "Google").Build(),
 			expectedMatch: false,
 		},
 		{
@@ -97,8 +92,7 @@ func TestMatches(t *testing.T) {
 					},
 				},
 			},
-			clusterlabels: map[string]string{"cloud": "Amazon"},
-			clusterclaims: map[string]string{"region": "us-east-1"},
+			cluster:       testinghelpers.NewManagedCluster("test").WithLabel("cloud", "Amazon").WithClaim("region", "us-east-1").Build(),
 			expectedMatch: true,
 		},
 		{
@@ -119,21 +113,68 @@ func TestMatches(t *testing.T) {
 					},
 				},
 			},
-			clusterlabels: map[string]string{"region": "us-east-1"},
-			clusterclaims: map[string]string{"cloud": "Amazon"},
+			cluster:       testinghelpers.NewManagedCluster("test").WithLabel("region", "us-east-1").WithClaim("cloud", "Amazon").Build(),
+			expectedMatch: false,
+		},
+		{
+			name: "match with cel expression - label",
+			clusterselector: clusterapiv1beta1.ClusterSelector{
+				CelSelector: clusterapiv1beta1.ClusterCelSelector{
+					CelExpressions: []string{
+						`managedCluster.metadata.labels["version"].matches('^1\\.(14|15)\\.\\d+$')`,
+					},
+				},
+			},
+			cluster:       testinghelpers.NewManagedCluster("test").WithLabel("version", "1.14.3").Build(),
+			expectedMatch: true,
+		},
+		{
+			name: "not match with cel expression - label",
+			clusterselector: clusterapiv1beta1.ClusterSelector{
+				CelSelector: clusterapiv1beta1.ClusterCelSelector{
+					CelExpressions: []string{
+						`managedCluster.metadata.labels["version"].matches('^1\\.(14|15)\\.\\d+$')`,
+					},
+				},
+			},
+			cluster:       testinghelpers.NewManagedCluster("test").WithLabel("version", "1.16.3").Build(),
+			expectedMatch: false,
+		},
+		{
+			name: "match with cel expression - claim",
+			clusterselector: clusterapiv1beta1.ClusterSelector{
+				CelSelector: clusterapiv1beta1.ClusterCelSelector{
+					CelExpressions: []string{
+						`managedCluster.status.clusterClaims.exists(c, c.name == "version" && c.value.matches('^1\\.(14|15)\\.\\d+$'))`,
+					},
+				},
+			},
+			cluster:       testinghelpers.NewManagedCluster("test").WithClaim("version", "1.14.3").Build(),
+			expectedMatch: true,
+		},
+		{
+			name: "not match with cel expression - claim",
+			clusterselector: clusterapiv1beta1.ClusterSelector{
+				CelSelector: clusterapiv1beta1.ClusterCelSelector{
+					CelExpressions: []string{
+						`managedCluster.status.clusterClaims.exists(c, c.name == "version" && c.value.matches('^1\\.(14|15)\\.\\d+$'))`,
+					},
+				},
+			},
+			cluster:       testinghelpers.NewManagedCluster("test").WithClaim("version", "1.16.3").Build(),
 			expectedMatch: false,
 		},
 	}
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			clusterSelector, err := NewClusterSelector(c.clusterselector)
+			clusterSelector, err := NewClusterSelector(c.clusterselector, nil)
 			if err != nil {
 				t.Errorf("unexpected err: %v", err)
 			}
-			result := clusterSelector.Matches(c.clusterlabels, c.clusterclaims)
+			result := clusterSelector.Matches(c.cluster)
 			if c.expectedMatch != result {
-				t.Errorf("expected match to be %v but get : %v", c.expectedMatch, result)
+				t.Errorf("expected match to be %v but got: %v", c.expectedMatch, result)
 			}
 		})
 	}

pkg/placement/helpers/testing/builders.go

@@ -107,11 +107,11 @@ func (b *PlacementBuilder) WithDeletionTimestamp() *PlacementBuilder {
 	return b
 }
 
-func (b *PlacementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1beta1.ClusterClaimSelector) *PlacementBuilder {
+func (b *PlacementBuilder) AddPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1beta1.ClusterClaimSelector, celSelector *clusterapiv1beta1.ClusterCelSelector) *PlacementBuilder {
 	if b.placement.Spec.Predicates == nil {
 		b.placement.Spec.Predicates = []clusterapiv1beta1.ClusterPredicate{}
 	}
-	b.placement.Spec.Predicates = append(b.placement.Spec.Predicates, NewClusterPredicate(labelSelector, claimSelector))
+	b.placement.Spec.Predicates = append(b.placement.Spec.Predicates, NewClusterPredicate(labelSelector, claimSelector, celSelector))
 	return b
 }
 
@@ -169,7 +169,7 @@ func (b *PlacementBuilder) Build() *clusterapiv1beta1.Placement {
 	return b.placement
 }
 
-func NewClusterPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1beta1.ClusterClaimSelector) clusterapiv1beta1.ClusterPredicate {
+func NewClusterPredicate(labelSelector *metav1.LabelSelector, claimSelector *clusterapiv1beta1.ClusterClaimSelector, celSelector *clusterapiv1beta1.ClusterCelSelector) clusterapiv1beta1.ClusterPredicate {
 	predicate := clusterapiv1beta1.ClusterPredicate{
 		RequiredClusterSelector: clusterapiv1beta1.ClusterSelector{},
 	}
@@ -182,6 +182,10 @@ func NewClusterPredicate(labelSelector *metav1.LabelSelector, claimSelector *clu
 		predicate.RequiredClusterSelector.ClaimSelector = *claimSelector
 	}
 
+	if celSelector != nil {
+		predicate.RequiredClusterSelector.CelSelector = *celSelector
+	}
+
 	return predicate
 }
 
@@ -309,6 +313,11 @@ func (b *ManagedClusterBuilder) WithDeletionTimestamp() *ManagedClusterBuilder {
 	return b
 }
 
+func (b *ManagedClusterBuilder) Withk8sVersion(version string) *ManagedClusterBuilder {
+	b.cluster.Status.Version.Kubernetes = version
+	return b
+}
+
 func (b *ManagedClusterBuilder) Build() *clusterapiv1.ManagedCluster {
 	return b.cluster
 }
@@ -381,6 +390,17 @@ func (a *AddOnPlacementScoreBuilder) WithValidUntil(validUntil time.Time) *AddOn
 	return a
 }
 
+func (a *AddOnPlacementScoreBuilder) WithQuantity(name string, quantity string) *AddOnPlacementScoreBuilder {
+	for i, score := range a.addOnPlacementScore.Status.Scores {
+		if score.Name == name {
+			q, _ := resource.ParseQuantity(quantity)
+			a.addOnPlacementScore.Status.Scores[i].Quantity = q
+			return a
+		}
+	}
+	return a
+}
+
 func (a *AddOnPlacementScoreBuilder) Build() *clusterapiv1alpha1.AddOnPlacementScore {
 	return a.addOnPlacementScore
 }

pkg/placement/plugins/predicate/predicate.go

@@ -6,6 +6,7 @@ import (
 
 	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
 	clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
+	mclcel "open-cluster-management.io/sdk-go/pkg/cel/managedcluster"
 
 	"open-cluster-management.io/ocm/pkg/placement/controllers/framework"
 	"open-cluster-management.io/ocm/pkg/placement/helpers"
@@ -16,10 +17,12 @@ var _ plugins.Filter = &Predicate{}
 
 const description = "Predicate filter filters the clusters based on predicate defined in placement"
 
-type Predicate struct{}
+type Predicate struct {
+	handle plugins.Handle
+}
 
 func New(handle plugins.Handle) *Predicate {
-	return &Predicate{}
+	return &Predicate{handle: handle}
 }
 
 func (p *Predicate) Name() string {
@@ -39,16 +42,17 @@ func (p *Predicate) Filter(
 			Filtered: clusters,
 		}, status
 	}
-	if len(clusters) == 0 {
-		return plugins.PluginFilterResult{
-			Filtered: clusters,
-		}, status
+
+	// Create CEL evaluator once for all predicates
+	celEvaluator, err := mclcel.NewManagedClusterEvaluator(p.handle.ScoreLister())
+	if err != nil {
+		return plugins.PluginFilterResult{}, framework.NewStatus(p.Name(), framework.Misconfigured, err.Error())
 	}
 
-	// prebuild label/claim selectors for each predicate
+	// prebuild cluster selectors for each predicate
 	clusterSelectors := []*helpers.ClusterSelector{}
 	for _, predicate := range placement.Spec.Predicates {
-		clusterSelector, err := helpers.NewClusterSelector(predicate.RequiredClusterSelector)
+		clusterSelector, err := helpers.NewClusterSelector(predicate.RequiredClusterSelector, celEvaluator)
 		if err != nil {
 			return plugins.PluginFilterResult{}, framework.NewStatus(
 				p.Name(),
@@ -62,9 +66,8 @@ func (p *Predicate) Filter(
 	// match cluster with selectors one by one
 	matched := []*clusterapiv1.ManagedCluster{}
 	for _, cluster := range clusters {
-		claims := helpers.GetClusterClaims(cluster)
 		for _, cs := range clusterSelectors {
-			if ok := cs.Matches(cluster.Labels, claims); !ok {
+			if ok := cs.Matches(cluster); !ok {
 				continue
 			}
 			matched = append(matched, cluster)