WIP: add a namespace selector to the provisioner spec

Makefile

@@ -52,6 +52,7 @@ apply: ## Deploy the controller into your ~/.kube/config cluster
 		$(HELM_OPTS) \
 		--set controller.image=$(shell $(WITH_GOFLAGS) ko build -B github.com/aws/karpenter/cmd/controller) \
 		--set webhook.image=$(shell $(WITH_GOFLAGS) ko build -B github.com/aws/karpenter/cmd/webhook)
+	kubectl apply -f charts/karpenter/crds/
 
 delete: ## Delete the controller from your ~/.kube/config cluster
 	helm uninstall karpenter --namespace karpenter

charts/karpenter/crds/karpenter.sh_provisioners.yaml

@@ -71,6 +71,63 @@ spec:
                       that Karpenter supports for limiting.
                     type: object
                 type: object
+              namespaceSelector:
+                description: A label query over the set of namespaces that the provisioner
+                  applies to. The provisioner is applied to the union of the namespaces
+                  selected by this field and the ones listed in the namespaces field.
+                  null selector and null or empty namespaces list means provision
+                  for all namespaces An empty selector ({}) matches all namespaces.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: A label selector requirement is a selector that
+                        contains values, a key, and an operator that relates the key
+                        and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: operator represents a key's relationship to
+                            a set of values. Valid operators are In, NotIn, Exists
+                            and DoesNotExist.
+                          type: string
+                        values:
+                          description: values is an array of string values. If the
+                            operator is In or NotIn, the values array must be non-empty.
+                            If the operator is Exists or DoesNotExist, the values
+                            array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: matchLabels is a map of {key,value} pairs. A single
+                      {key,value} in the matchLabels map is equivalent to an element
+                      of matchExpressions, whose key field is "key", the operator
+                      is "In", and the values array contains only "value". The requirements
+                      are ANDed.
+                    type: object
+                type: object
+              namespaces:
+                description: namespaces specifies a static list of namespace names
+                  that the provisioner applies to. The provisioner is applied to the
+                  union of the namespaces listed in this field and the ones selected
+                  by namespaceSelector. null or empty namespaces list and null namespaceSelector
+                  means provision for all namespaces
+                items:
+                  type: string
+                type: array
               provider:
                 description: Provider contains fields specific to your cloudprovider.
                 type: object

charts/karpenter/templates/clusterrole.yaml

@@ -30,6 +30,9 @@ rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["create"]
+  - apiGroups: [ "" ]
+    resources: [ "namespaces" ]
+    verbs: [ "get", "list", "watch" ]
   - apiGroups: [""]
     resources: ["pods/binding", "pods/eviction"]
     verbs: ["create"]

pkg/apis/provisioning/v1alpha5/provisioner.go

@@ -25,6 +25,19 @@ import (
 type ProvisionerSpec struct {
 	// Constraints are applied to all nodes launched by this provisioner.
 	Constraints `json:",inline"`
+	// namespaces specifies a static list of namespace names that the provisioner applies to.
+	// The provisioner is applied to the union of the namespaces listed in this field
+	// and the ones selected by namespaceSelector.
+	// null or empty namespaces list and null namespaceSelector means provision for all namespaces
+	// +optional
+	Namespaces []string `json:"namespaces,omitempty" protobuf:"bytes,2,rep,name=namespaces"`
+	// A label query over the set of namespaces that the provisioner applies to.
+	// The provisioner is applied to the union of the namespaces selected by this field
+	// and the ones listed in the namespaces field.
+	// null selector and null or empty namespaces list means provision for all namespaces
+	// An empty selector ({}) matches all namespaces.
+	// +optional
+	NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty" protobuf:"bytes,4,opt,name=namespaceSelector"`
 	// TTLSecondsAfterEmpty is the number of seconds the controller will wait
 	// before attempting to delete a node, measured from when the node is
 	// detected to be empty. A Node is considered to be empty when it does not

pkg/controllers/selection/controller.go

@@ -19,6 +19,9 @@ import (
 	"fmt"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+
 	"github.com/go-logr/zapr"
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
@@ -75,6 +78,7 @@ func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reco
 		logging.FromContext(ctx).Errorf("Ignoring pod, %s", err)
 		return reconcile.Result{}, nil
 	}
+
 	// Select a provisioner, wait for it to bind the pod, and verify scheduling succeeded in the next loop
 	if err := c.selectProvisioner(ctx, pod); err != nil {
 		logging.FromContext(ctx).Debugf("Could not schedule pod, %s", err)
@@ -96,13 +100,29 @@ func (c *Controller) selectProvisioner(ctx context.Context, pod *v1.Pod) (errs e
 	if len(provisioners) == 0 {
 		return nil
 	}
+
+	// lookup the pod namespace for matching against the provisioner
+	var podNamespace v1.Namespace
+	if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: pod.Namespace}, &podNamespace); err != nil {
+		return err
+	}
+
 	for _, candidate := range c.provisioners.List(ctx) {
+		// check if the provisioner is allowed to provision pods in this namespace
+		if err := validateNamespace(candidate, podNamespace); err != nil {
+			errs = multierr.Append(errs, fmt.Errorf("tried provisioner/%s: %w", candidate.Name, err))
+			continue
+		}
+
+		// ValidatePod is on Constraints, which is embedded in ProvisionerSpec.  If that gets reworked, consider moving
+		// validateNamespace to there as well
 		if err := candidate.Spec.DeepCopy().ValidatePod(pod); err != nil {
 			errs = multierr.Append(errs, fmt.Errorf("tried provisioner/%s: %w", candidate.Name, err))
-		} else {
-			provisioner = candidate
-			break
+			continue
 		}
+
+		provisioner = candidate
+		break
 	}
 	if provisioner == nil {
 		return fmt.Errorf("matched 0/%d provisioners, %w", len(multierr.Errors(errs)), errs)
@@ -114,6 +134,35 @@ func (c *Controller) selectProvisioner(ctx context.Context, pod *v1.Pod) (errs e
 	return nil
 }
 
+// validateNamespace returns nil if the candidate provisioner is configured to provision pods in the provided
+// namespace
+func validateNamespace(candidate *provisioning.Provisioner, namespace v1.Namespace) error {
+	// no namespace list or label selector provided, so everything passes
+	if len(candidate.Spec.Namespaces) == 0 && candidate.Spec.NamespaceSelector == nil {
+		return nil
+	}
+
+	// the namespace of the pod must match one of the list of namespaces or the selector
+	for _, ns := range candidate.Spec.Namespaces {
+		if ns == namespace.Name {
+			return nil
+		}
+	}
+
+	// For an undefined namespace selector, the selector itself matches nothing.  This
+	// provides the desired semantics here as we know there is either a namespace list
+	// or a namespace label selector and the namespace has already failed to match the
+	// possibly empty list
+	selector, err := metav1.LabelSelectorAsSelector(candidate.Spec.NamespaceSelector)
+	if err != nil {
+		return err
+	}
+	if !selector.Matches(labels.Set(namespace.Labels)) {
+		return fmt.Errorf("doesn't match namespaces being provisioned")
+	}
+	return nil
+}
+
 func isProvisionable(p *v1.Pod) bool {
 	return !pod.IsScheduled(p) &&
 		!pod.IsPreempting(p) &&

pkg/controllers/selection/suite_test.go

@@ -76,6 +76,151 @@ var _ = AfterEach(func() {
 	ExpectProvisioningCleanedUp(ctx, env.Client, provisioners)
 })
 
+var _ = Describe("Namespace Selector", func() {
+	It("should schedule if there is no namespace selector", func() {
+		provisioner.Spec.NamespaceSelector = nil
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+	It("should schedule if there is an empty namespace selector", func() {
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels:      map[string]string{},
+			MatchExpressions: []metav1.LabelSelectorRequirement{},
+		}
+
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+	It("should not schedule if the pod isn't in a matching namespace, namespace list", func() {
+		provisioner.Spec.Namespaces = []string{"foo"}
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(),
+		)[0]
+		ExpectNotScheduled(ctx, env.Client, pod)
+	})
+	It("should not schedule if the pod isn't in a matching namespace, MatchLabels", func() {
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"foo": "bar",
+			},
+		}
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(),
+		)[0]
+		ExpectNotScheduled(ctx, env.Client, pod)
+	})
+	It("should not schedule if the pod isn't in a matching namespace, MatchExpressions", func() {
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{
+					Key:      "foo",
+					Operator: metav1.LabelSelectorOpIn,
+					Values:   []string{"bar"},
+				},
+			},
+		}
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(),
+		)[0]
+		ExpectNotScheduled(ctx, env.Client, pod)
+	})
+	It("should schedule if the pod is in a matching namespace, namespace list", func() {
+		ns := randomdata.Noun() + randomdata.Adjective()
+		ExpectCreated(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})
+		provisioner.Spec.Namespaces = []string{ns}
+
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Namespace: ns}}),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+	It("should schedule if the pod is in a matching namespace, MatchLabels", func() {
+		ns := randomdata.Noun() + randomdata.Adjective() // need a lowercase name here
+		ExpectCreated(ctx, env.Client, &v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   ns,
+				Labels: map[string]string{"foo": "bar"},
+			},
+		})
+
+		// select for namespaces with the label foo=bar
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels: map[string]string{"foo": "bar"},
+		}
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Namespace: ns}}),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+	It("should schedule if the pod is in a matching namespace, MatchExpressions", func() {
+		ns := randomdata.Noun() + randomdata.Adjective()
+		ExpectCreated(ctx, env.Client, &v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   ns,
+				Labels: map[string]string{"foo": "bar"},
+			},
+		})
+
+		// select for namespaces with the label foo in ["bar"]
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{
+					Key:      "foo",
+					Operator: metav1.LabelSelectorOpIn,
+					Values:   []string{"bar"},
+				},
+			},
+		}
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Namespace: ns}}),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+	It("should schedule if the pod is in a matching namespace list but fails selector ", func() {
+		ns := randomdata.Noun() + randomdata.Adjective()
+		ExpectCreated(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})
+		provisioner.Spec.Namespaces = []string{ns}
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels: map[string]string{"foo": "bar"},
+		}
+
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Namespace: ns}}),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+	It("should schedule if the pod is not in a matching namespace list but passes selector", func() {
+		ns := randomdata.Noun() + randomdata.Adjective() // need a lowercase name here
+		ExpectCreated(ctx, env.Client, &v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   ns,
+				Labels: map[string]string{"foo": "bar"},
+			},
+		})
+
+		// will fail the namespaec list match, but pass the selector
+		provisioner.Spec.Namespaces = []string{"somethingelse"}
+		provisioner.Spec.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels: map[string]string{"foo": "bar"},
+		}
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
+			test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Namespace: ns}}),
+		)[0]
+		ExpectScheduled(ctx, env.Client, pod)
+	})
+})
+
 var _ = Describe("Volume Topology Requirements", func() {
 	var storageClass *storagev1.StorageClass
 	BeforeEach(func() {