Merge pull request #668 from mwielgus/integrations

Config option to select which job framework integrations should be enabled
kubernetes-sigs · Apr 4, 2023 · 5ae0353 · 5ae0353
2 parents 1598204 + 1a5b142
commit 5ae0353
Show file tree

Hide file tree

Showing 13 changed files with 301 additions and 69 deletions.
diff --git a/apis/config/v1beta1/configuration_types.go b/apis/config/v1beta1/configuration_types.go
@@ -55,6 +55,10 @@ type Configuration struct {
 	// ClientConnection provides additional configuration options for Kubernetes
 	// API server client.
 	ClientConnection *ClientConnection `json:"clientConnection,omitempty"`
+
+	// Integrations provide configuration options for AI/ML/Batch frameworks
+	// integrations (including K8S job).
+	Integrations *Integrations `json:"integrations,omitempty"`
 }
 
 type WaitForPodsReady struct {
@@ -96,3 +100,11 @@ type ClientConnection struct {
 	// Burst allows extra queries to accumulate when a client is exceeding its rate.
 	Burst *int32 `json:"burst,omitempty"`
 }
+
+type Integrations struct {
+	// List of framework names to be enabled.
+	// Possible options:
+	//  - "batch/job"
+	//  - "kubeflow.org/mpijob"
+	Frameworks []string `json:"frameworks,omitempty"`
+}
diff --git a/apis/config/v1beta1/defaults.go b/apis/config/v1beta1/defaults.go
@@ -19,10 +19,11 @@ package v1beta1
 import (
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/utils/pointer"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/kueue/pkg/controller/jobs/job"
 )
 
 const (
@@ -89,4 +90,10 @@ func SetDefaults_Configuration(cfg *Configuration) {
 	if cfg.WaitForPodsReady != nil && cfg.WaitForPodsReady.Timeout == nil {
 		cfg.WaitForPodsReady.Timeout = &metav1.Duration{Duration: defaultPodsReadyTimeout}
 	}
+	if cfg.Integrations == nil {
+		cfg.Integrations = &Integrations{}
+	}
+	if cfg.Integrations.Frameworks == nil {
+		cfg.Integrations.Frameworks = []string{job.FrameworkName}
+	}
 }
diff --git a/apis/config/v1beta1/defaults_test.go b/apis/config/v1beta1/defaults_test.go
@@ -25,6 +25,8 @@ import (
 	componentconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
 	"k8s.io/utils/pointer"
 	ctrlconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+
+	"sigs.k8s.io/kueue/pkg/controller/jobs/job"
 )
 
 const (
@@ -51,6 +53,9 @@ func TestSetDefaults_Configuration(t *testing.T) {
 		QPS:   pointer.Float32(DefaultClientConnectionQPS),
 		Burst: pointer.Int32(DefaultClientConnectionBurst),
 	}
+	defaultIntegrations := &Integrations{
+		Frameworks: []string{job.FrameworkName},
+	}
 	podsReadyTimeoutTimeout := metav1.Duration{Duration: defaultPodsReadyTimeout}
 	podsReadyTimeoutOverwrite := metav1.Duration{Duration: time.Minute}
 
@@ -71,6 +76,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"defaulting ControllerManagerConfigurationSpec": {
@@ -105,6 +111,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"should not default ControllerManagerConfigurationSpec": {
@@ -127,6 +134,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: pointer.Bool(false),
 				},
+				Integrations: defaultIntegrations,
 			},
 			want: &Configuration{
 				Namespace: pointer.String(DefaultNamespace),
@@ -149,6 +157,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"should not set LeaderElectionID": {
@@ -182,6 +191,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"defaulting InternalCertManagement": {
@@ -197,6 +207,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					WebhookSecretName:  pointer.String(DefaultWebhookSecretName),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"should not default InternalCertManagement": {
@@ -213,6 +224,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"should not default values in custom ClientConnection": {
@@ -236,6 +248,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					QPS:   pointer.Float32(123.0),
 					Burst: pointer.Int32(456),
 				},
+				Integrations: defaultIntegrations,
 			},
 		},
 		"should default empty custom ClientConnection": {
@@ -253,6 +266,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"defaulting waitForPodsReady.timeout": {
@@ -275,6 +289,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
 			},
 		},
 		"respecting provided waitForPodsReady.timeout": {
@@ -298,6 +313,28 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Enable: pointer.Bool(false),
 				},
 				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
+			},
+		},
+		"integrations": {
+			original: &Configuration{
+				InternalCertManagement: &InternalCertManagement{
+					Enable: pointer.Bool(false),
+				},
+				Integrations: &Integrations{
+					Frameworks: []string{"a", "b"},
+				},
+			},
+			want: &Configuration{
+				Namespace:                          pointer.String(DefaultNamespace),
+				ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
+				InternalCertManagement: &InternalCertManagement{
+					Enable: pointer.Bool(false),
+				},
+				ClientConnection: defaultClientConnection,
+				Integrations: &Integrations{
+					Frameworks: []string{"a", "b"},
+				},
 			},
 		},
 	}

diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go
diff --git a/config/components/manager/controller_manager_config.yaml b/config/components/manager/controller_manager_config.yaml
@@ -27,3 +27,7 @@ clientConnection:
 #  enable: false
 #  webhookServiceName: ""
 #  webhookSecretName: ""
+integrations:
+  frameworks:
+  - "batch/job"
+# - "kubeflow.org/mpijob"
diff --git a/config/components/webhook/manifests.yaml b/config/components/webhook/manifests.yaml
@@ -11,28 +11,28 @@ webhooks:
     service:
       name: webhook-service
       namespace: system
-      path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue
+      path: /mutate-batch-v1-job
   failurePolicy: Fail
-  name: mclusterqueue.kb.io
+  name: mjob.kb.io
   rules:
   - apiGroups:
-    - kueue.x-k8s.io
+    - batch
     apiVersions:
-    - v1beta1
+    - v1
     operations:
     - CREATE
     resources:
-    - clusterqueues
+    - jobs
   sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
     service:
       name: webhook-service
       namespace: system
-      path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor
+      path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue
   failurePolicy: Fail
-  name: mresourceflavor.kb.io
+  name: mclusterqueue.kb.io
   rules:
   - apiGroups:
     - kueue.x-k8s.io
@@ -41,46 +41,46 @@ webhooks:
     operations:
     - CREATE
     resources:
-    - resourceflavors
+    - clusterqueues
   sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
     service:
       name: webhook-service
       namespace: system
-      path: /mutate-kueue-x-k8s-io-v1beta1-workload
+      path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor
   failurePolicy: Fail
-  name: mworkload.kb.io
+  name: mresourceflavor.kb.io
   rules:
   - apiGroups:
     - kueue.x-k8s.io
     apiVersions:
     - v1beta1
     operations:
     - CREATE
-    - UPDATE
     resources:
-    - workloads
+    - resourceflavors
   sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
     service:
       name: webhook-service
       namespace: system
-      path: /mutate-batch-v1-job
+      path: /mutate-kueue-x-k8s-io-v1beta1-workload
   failurePolicy: Fail
-  name: mjob.kb.io
+  name: mworkload.kb.io
   rules:
   - apiGroups:
-    - batch
+    - kueue.x-k8s.io
     apiVersions:
-    - v1
+    - v1beta1
     operations:
     - CREATE
+    - UPDATE
     resources:
-    - jobs
+    - workloads
   sideEffects: None
 - admissionReviewVersions:
   - v1
@@ -108,6 +108,25 @@ metadata:
   creationTimestamp: null
   name: validating-webhook-configuration
 webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-batch-v1-job
+  failurePolicy: Fail
+  name: vjob.kb.io
+  rules:
+  - apiGroups:
+    - batch
+    apiVersions:
+    - v1
+    operations:
+    - UPDATE
+    resources:
+    - jobs
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
@@ -189,25 +208,6 @@ webhooks:
     - workloads
     - workloads/status
   sideEffects: None
-- admissionReviewVersions:
-  - v1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate-batch-v1-job
-  failurePolicy: Fail
-  name: vjob.kb.io
-  rules:
-  - apiGroups:
-    - batch
-    apiVersions:
-    - v1
-    operations:
-    - UPDATE
-    resources:
-    - jobs
-  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig: