Merge pull request #28 from cwiklik/add-combinedstatus-ctrl

:bug Parameterize clustermetrics image

.pre-commit-config.yaml

@@ -8,3 +8,4 @@ repos:
     -   id: end-of-file-fixer
 #    -   id: check-yaml
     -   id: check-added-large-files
+        args: ["--maxkb=2000"]

charts/cluster-metrics/templates/operator.yaml

@@ -297,7 +297,7 @@ spec:
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
         - --metrics-name={{.Values.clusterName}}
-        image: ko.local/cluster-metrics:149d115
+        image: ko.local/cluster-metrics:c8527e5
         livenessProbe:
           httpGet:
             path: /healthz

charts/kueue-ks/templates/operator.yaml

@@ -611,6 +611,8 @@ spec:
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
         - --deleteOnCompletion={{.Values.deleteOnCompletion}}
+        - --clusterMetricsImage={{.Values.clusterMetricsImage}}
+        - --clusterMetricsImageTag={{.Values.clusterMetricsImageTag}}
         image: {{.Values.image}}
         livenessProbe:
           httpGet:

charts/kueue-ks/values.yaml

@@ -1,5 +1,6 @@
 deleteOnCompletion: false
-
+clusterMetricsImage: ghcr.io/cwiklik/clustermetrics
+clusterMetricsImageTag: v1
 service:
   type: ClusterIP
   port: 80

clustermetrics/Makefile

@@ -21,6 +21,7 @@ CLUSTER ?= kubeflex
 HELM_OPTS ?= --set clusterName=local
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.29.0
+GHCR_IMG=ghcr.io/cwiklik/clustermetrics:v1
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -119,6 +120,16 @@ docker-build: ## Build docker image with the manager.
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}
 
+# If you wish to build the manager image targeting other platforms you can use the --platform flag.
+# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
+# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+.PHONY: docker-ghcr-build
+docker-ghcr-build: ## Build docker image with the manager.
+	$(CONTAINER_TOOL) build -t ${GHCR_IMG} .
+
+.PHONY: docker-ghcr-push
+docker-ghcr-push: ## Push docker image with the manager.
+	$(CONTAINER_TOOL) push ${GHCR_IMG}
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
 # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
@@ -136,6 +147,16 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 	- $(CONTAINER_TOOL) buildx rm project-v3-builder
 	rm Dockerfile.cross
 
+.PHONY: docker-ghcr-buildx
+docker-ghcr-buildx: ## Build and push docker image for the manager for cross-platform support
+	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	- $(CONTAINER_TOOL) buildx create --name project-v3-builder
+	$(CONTAINER_TOOL) buildx use project-v3-builder
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${GHCR_IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx rm project-v3-builder
+	rm Dockerfile.cross
+
 .PHONY: build-installer
 build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.
 	mkdir -p dist

clustermetrics/config/crd/bases/galaxy.kubestellar.io_clustermetrics.yaml

@@ -5,6 +5,8 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.14.0
   name: clustermetrics.galaxy.kubestellar.io
+  labels:
+    app.kubernetes.io/name: clustermetrics
 spec:
   group: galaxy.kubestellar.io
   names:

clustermetrics/config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ko.local/cluster-metrics
-  newTag: 149d115
+  newTag: c8527e5

kueue-ks/Makefile

@@ -6,10 +6,14 @@ IMAGE_TAG ?= $(shell git rev-parse --short HEAD)
 CMD_NAME ?= kueue-ks
 IMG ?= ${KO_DOCKER_REPO}/${CMD_NAME}:${IMAGE_TAG}
 HELM_IMG := --set image=${IMG}
-HELM_OPTS := ${HELM_OPTS} ${HELM_IMG}
-@echo ${HELM_OPTS}
+#CLUSTER_METRICS_IMAGE ?= --set clusterMetricsImage=ghcr.io/cwiklik/clustermetrics
+#CLUSTER_METRICS_IMAGE_TAG ?= --set clusterMetricsImageTag=v1
+HELM_OPTS ?= ${HELM_IMG}
+#${CLUSTER_METRICS_IMAGE} ${CLUSTER_METRICS_IMAGE_TAG}
+echo ${HELM_OPTS}
 
 #FOO ?= " --set image=${IMG}"
+
 # Image URL to use all building/pushing image targets
 #IMG ?= controller:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.

kueue-ks/cmd/kueue-ks/main.go

@@ -30,6 +30,7 @@ import (
 	kfv1aplha1 "github.com/kubestellar/kubeflex/api/v1alpha1"
 	ksv1alpha1 "github.com/kubestellar/kubestellar/api/control/v1alpha1"
 	kslclient "github.com/kubestellar/kubestellar/pkg/client"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -42,8 +43,6 @@ import (
 	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/utils/clock"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -92,6 +91,8 @@ func main() {
 	var probeAddr string
 	var enableHTTP2 bool
 	var clusterQueue string
+	var clusterMetricsImage string
+	var clusterMetricsImageTag string
 	var deleteOnCompletion bool
 	var defaultResourceFlavorName string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8085", "The address the metric endpoint binds to.")
@@ -104,6 +105,9 @@ func main() {
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.StringVar(&clusterQueue, "clusterQueue-name", "cluster-queue-ks", "cluster queue name")
+	flag.StringVar(&clusterMetricsImage, "clusterMetricsImage", "", "clustermetrics image")
+	flag.StringVar(&clusterMetricsImageTag, "clusterMetricsImageTag", "", "clustermetrics image tag")
+
 	flag.BoolVar(&deleteOnCompletion, "deleteOnCompletion", false, "If set, workload will be auto cleaned up from the WEC")
 	flag.StringVar(&defaultResourceFlavorName, "defaultResourceFlavorName", "default-flavor", "default flavor name")
 	opts := zap.Options{
@@ -113,6 +117,7 @@ func main() {
 	flag.Parse()
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -123,6 +128,8 @@ func main() {
 		setupLog.Info("disabling http/2")
 		c.NextProtos = []string{"http/1.1"}
 	}
+	clusterMetricsImage = clusterMetricsImage + ":" + clusterMetricsImageTag
+	fmt.Printf("---------------- clusterMetricsImage %v\n", clusterMetricsImage)
 
 	enableLeaderElection = false
 	tlsOpts := []func(*tls.Config){}
@@ -133,6 +140,18 @@ func main() {
 		TLSOpts: tlsOpts,
 	})
 
+	wdsConfig, _, err := getRestConfig("wds1", "wds")
+	if err != nil {
+		setupLog.Error(err, "error while calling getRestConfig for wds1")
+		os.Exit(1)
+	}
+
+	itsConfig, _, err := getRestConfig("its1", "its")
+	if err != nil {
+		setupLog.Error(err, "error while calling getRestConfig for its1")
+		os.Exit(1)
+	}
+
 	kflexConfig, err := rest.InClusterConfig()
 	if err != nil {
 		setupLog.Error(err, "unable to get kubeconfig")
@@ -160,6 +179,64 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
+	probeAddr = ":8084"
+	metricsAddr = ":8089"
+
+	its1Mgr, err := manager.New(itsConfig,
+		ctrl.Options{
+			Scheme: scheme,
+			Metrics: metricsserver.Options{
+				BindAddress:   metricsAddr,
+				SecureServing: secureMetrics,
+				TLSOpts:       tlsOpts,
+			},
+			WebhookServer:          webhookServer,
+			HealthProbeBindAddress: probeAddr,
+			LeaderElection:         enableLeaderElection,
+			LeaderElectionID:       "423ebda8.kubestellar.io",
+		},
+	)
+	if err != nil {
+		setupLog.Error(err, "unable to start its1 manager")
+
+		os.Exit(1)
+	}
+
+	probeAddr = ":8083"
+	metricsAddr = ":8088"
+
+	wds1Mgr, err := manager.New(wdsConfig,
+		ctrl.Options{
+			Scheme: scheme,
+			Metrics: metricsserver.Options{
+				BindAddress:   metricsAddr,
+				SecureServing: secureMetrics,
+				TLSOpts:       tlsOpts,
+			},
+			WebhookServer:          webhookServer,
+			HealthProbeBindAddress: probeAddr,
+			LeaderElection:         enableLeaderElection,
+			LeaderElectionID:       "423ebda8.kubestellar.io",
+		},
+	)
+	if err != nil {
+		setupLog.Error(err, "unable to start wds1 manager")
+
+		os.Exit(1)
+	}
+
+	//setupReconciller(its1Mgr, wds1Mgr, kflexMgr)
+	if err = (&controller.ManagedClusterReconciler{
+		Client:              its1Mgr.GetClient(),
+		Scheme:              its1Mgr.GetScheme(),
+		Its1Client:          its1Mgr.GetClient(),
+		Wds1Client:          wds1Mgr.GetClient(),
+		KubeflexClient:      kflexMgr.GetClient(),
+		ClusterMetricsImage: clusterMetricsImage,
+	}).SetupWithManager(its1Mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "ManagedCluster")
+		os.Exit(1)
+	}
 
 	if err = (&controller.ClusterMetricsReconciler{
 		Client:                    kflexMgr.GetClient(),
@@ -210,24 +287,12 @@ func main() {
 		Scheduler:              scheduler.NewDefaultScheduler(),
 		Recorder:               kflexMgr.GetEventRecorderFor("job-recorder"),
 		CleanupWecOnCompletion: deleteOnCompletion,
+		Clock:                  clock.RealClock{},
 	}
 	if err = wr.SetupWithManager(kflexMgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Workload")
 		os.Exit(1)
 	}
-	/*
-		if err = (&controller.WorkloadReconciler{
-			Client:        kflexMgr.GetClient(),
-			KueueClient:   kClient,
-			DynamicClient: dynClient,
-			RestMapper:    rm,
-			Scheduler:     scheduler.NewDefaultScheduler(),
-			Recorder:      kflexMgr.GetEventRecorderFor("job-recorder"),
-		}).SetupWithManager(kflexMgr); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "Workload")
-			os.Exit(1)
-		}
-	*/
 	if err = (&controller.CombinedStatusReconciler{
 		Client:             kflexMgr.GetClient(),
 		Scheme:             kflexMgr.GetScheme(),
@@ -240,6 +305,18 @@ func main() {
 
 	//+kubebuilder:scaffold:builder
 
+	go func() {
+		if err := its1Mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+			setupLog.Error(err, "problem running its1 manager")
+			os.Exit(1)
+		}
+	}()
+	go func() {
+		if err := wds1Mgr.Start(context.Background()); err != nil {
+			setupLog.Error(err, "problem running wds1 manager")
+			os.Exit(1)
+		}
+	}()
 	go func() {
 		if err := kflexMgr.Start(k8sctx); err != nil {
 			setupLog.Error(err, "problem running kubeflex manager")
@@ -251,11 +328,12 @@ func main() {
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func setupReconciller(its1Mgr ctrl.Manager, kflexMgr ctrl.Manager) error {
+func setupReconciller(its1Mgr ctrl.Manager, wds1Mgr ctrl.Manager, kflexMgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(kflexMgr).
 		Complete(&controller.ManagedClusterReconciler{
 			Scheme:         its1Mgr.GetScheme(),
 			Its1Client:     its1Mgr.GetClient(),
+			Wds1Client:     wds1Mgr.GetClient(),
 			KubeflexClient: kflexMgr.GetClient(),
 		})
 }

kueue-ks/examples/tmp/batch-job-nosuspend.yaml → kueue-ks/examples/batch-job-test2.yaml

@@ -1,19 +1,21 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  generateName: sample-job-
+  generateName: h100-job-
   namespace: default
   labels:
-    kueue.x-k8s.io/queue-name: user-queue-ks
+    kueue.x-k8s.io/queue-name: user-queue
 spec:
   parallelism: 3
   completions: 3
   template:
     spec:
+      nodeSelector:
+        instance: h100
       containers:
       - name: dummy-job
         image: gcr.io/k8s-staging-perf-tests/sleep:v0.1.0
-        args: ["120s"]
+        args: ["50s"]
         resources:
           requests:
             cpu: 1

kueue-ks/examples/clustqtest.yaml

@@ -0,0 +1,15 @@
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ClusterQueue
+metadata:
+  name: "cluster-queue"
+spec:
+  namespaceSelector: {} # match all.
+  resourceGroups:
+  - coveredResources: ["cpu", "memory"]
+    flavors:
+    - name: "default-flavor"
+      resources:
+      - name: "cpu"
+        nominalQuota: 9
+      - name: "memory"
+        nominalQuota: 36Gi

kueue-ks/examples/default-flavor-test.yaml

@@ -0,0 +1,4 @@
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ResourceFlavor
+metadata:
+  name: "default-flavor"

kueue-ks/examples/localqtest.yaml

@@ -0,0 +1,7 @@
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: LocalQueue
+metadata:
+  namespace: "default"
+  name: "user-queue"
+spec:
+  clusterQueue: "cluster-queue"