Merge pull request #45 from flux-framework/add-sched-plugin

Add manifests and Fluence package from scheduler plugins

sig-scheduler-plugins/manifests/fluence/configmap.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scheduler-config
+  namespace: scheduler-plugins 
+data:
+  scheduler-config.yaml: |
+    apiVersion: kubescheduler.config.k8s.io/v1beta3
+    kind: KubeSchedulerConfiguration
+    leaderElection:
+      leaderElect: false
+    profiles:
+    - schedulerName: fluence
+      plugins:
+        preFilter:
+          enabled:
+          - name: Fluence
+        filter:
+          enabled:
+          - name: Fluence
+        score:
+          disabled:
+          - name: '*'

sig-scheduler-plugins/manifests/fluence/deploy.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fluence
+  namespace: scheduler-plugins
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: scheduler
+  template:
+    metadata:
+      labels:
+        component: scheduler
+    spec:
+      serviceAccountName: scheduler-plugins
+      containers:
+        - image: quay.io/cmisale1/fluence-sidecar:latest
+          imagePullPolicy: Always
+          command:
+          - /go/src/fluence/bin/server
+          - --policy=lonode
+          name: fluence-sidecar
+        - image: quay.io/cmisale1/fluence:dev
+          imagePullPolicy: Always
+          command:
+          - /bin/kube-scheduler
+          - --config=/etc/kubernetes/scheduler-config.yaml
+          - -v=9
+          name: fluence
+          resources:
+            requests:
+              cpu: '0.1'
+          securityContext:
+            privileged: false
+          volumeMounts:
+          - mountPath: /etc/kubernetes
+            name: scheduler-config
+      hostNetwork: false
+      hostPID: false
+      volumes:
+      - name: scheduler-config
+        configMap:
+          name: scheduler-config
+

sig-scheduler-plugins/manifests/fluence/rbac.yaml

@@ -0,0 +1,82 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins
+rules:
+- apiGroups: [""]
+  resources: ["namespaces", "configmaps"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "events.k8s.io"]
+  resources: ["events"]
+  verbs: ["create", "patch", "update"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create"]
+- apiGroups: ["coordination.k8s.io"]
+  resourceNames: ["kube-scheduler"]
+  resources: ["leases"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resourceNames: ["kube-scheduler"]
+  resources: ["endpoints"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "list", "watch", "patch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["delete", "get", "list", "watch", "update"]
+- apiGroups: [""]
+  resources: ["bindings", "pods/binding"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["patch", "update"]
+- apiGroups: [""]
+  resources: ["replicationcontrollers", "services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["persistentvolumeclaims", "persistentvolumes"]
+  verbs: ["get", "list", "watch", "patch", "update"]
+- apiGroups: ["authentication.k8s.io"]
+  resources: ["tokenreviews"]
+  verbs: ["create"]
+- apiGroups: ["authorization.k8s.io"]
+  resources: ["subjectaccessreviews"]
+  verbs: ["create"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["csinodes", "storageclasses" , "csidrivers" , "csistoragecapacities"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["topology.node.k8s.io"]
+  resources: ["noderesourcetopologies"]
+  verbs: ["*"]
+# resources need to be updated with the scheduler plugins used
+- apiGroups: ["scheduling.sigs.k8s.io"]
+  resources: ["podgroups", "elasticquotas"]
+  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins
+  namespace: scheduler-plugins
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: scheduler-plugins
+subjects:
+  - kind: ServiceAccount
+    name: scheduler-plugins
+    namespace: scheduler-plugins 

sig-scheduler-plugins/manifests/fluence/scheduling.sigs.k8s.io_podgroups.yaml

@@ -0,0 +1 @@
+../coscheduling/crd.yaml

sig-scheduler-plugins/manifests/fluence/serviceaccount.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: scheduler-plugins
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scheduler-plugins
+  namespace: scheduler-plugins

sig-scheduler-plugins/pkg/fluence/README.md

@@ -0,0 +1,29 @@
+# Overview
+
+Project to manage Flux tasks needed to standardize kubernetes HPC scheduling interfaces
+
+## Installing the chart
+
+More detail will be added here about installing the chart. You will
+be using the [install-as-a-second-scheduler](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/manifests/install/charts/as-a-second-scheduler)
+charts. Fluence-specific values are detailed below.
+
+### Fluence specific values
+
+In `values.yaml` it is possible to customize the container image, already defaulted to the latest release, and the allocation policy
+used by the scheduler.
+Most common options are:
+
+- `lonode`: choose the nodes with lower ID first. Can be compared to packing
+- `low`: choose cores with lowest IDs from multiple nodes. Can be compared to spread process-to-resource placement
+
+## Maturity Level
+
+<!-- Check one of the values: Sample, Alpha, Beta, GA -->
+
+- [x] Sample (for demonstrating and inspiring purpose)
+- [ ] Alpha (used in companies for pilot projects)
+- [ ] Beta (used in companies and developed actively)
+- [ ] Stable (used in companies for production workloads)
+
+<!-- TODO: write some useful KubeFlux documentation -->

sig-scheduler-plugins/pkg/fluence/core/core.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/scheduler/framework"
+	pb "sigs.k8s.io/scheduler-plugins/pkg/fluence/fluxcli-grpc"
+)
+
+type FluxStateData struct {
+	NodeName string
+}
+
+func (s *FluxStateData) Clone() framework.StateData {
+	clone := &FluxStateData{
+		NodeName: s.NodeName,
+	}
+	return clone
+}
+
+type NodePodsCount struct {
+	NodeName string
+	Count    int
+}
+
+var podgroupMap map[string][]NodePodsCount
+
+func Init() {
+	podgroupMap = make(map[string][]NodePodsCount, 0)
+}
+
+func (n *NodePodsCount) Clone() framework.StateData {
+	return &NodePodsCount{
+		NodeName: n.NodeName,
+		Count:    n.Count,
+	}
+}
+
+func CreateNodePodsList(nodelist []*pb.NodeAlloc, pgname string) (nodepods []NodePodsCount) {
+	nodepods = make([]NodePodsCount, len(nodelist))
+	for i, v := range nodelist {
+		nodepods[i] = NodePodsCount{
+			NodeName: v.GetNodeID(),
+			Count:    int(v.GetTasks()),
+		}
+	}
+	podgroupMap[pgname] = nodepods
+	klog.Info("MAP ", podgroupMap)
+
+	return
+}
+
+func HaveList(pgname string) bool {
+	_, exists := podgroupMap[pgname]
+	return exists
+}
+
+func GetNextNode(pgname string) (string, error) {
+	entry, ok := podgroupMap[pgname]
+	if !ok {
+		err := fmt.Errorf("Map is empty")
+		return "", err
+	}
+	if len(entry) == 0 {
+		err := fmt.Errorf("Error while getting a node")
+		return "", err
+	}
+
+	nodename := entry[0].NodeName
+
+	if entry[0].Count == 1 {
+		slice := entry[1:]
+		if len(slice) == 0 {
+			delete(podgroupMap, pgname)
+			return nodename, nil
+		}
+		podgroupMap[pgname] = slice
+		return nodename, nil
+	}
+	entry[0].Count = entry[0].Count - 1
+	return nodename, nil
+}