PoC: use PodSchedulingReadiness API to replace scheduler

.github/actions/deploy-klt-on-cluster/action.yml

@@ -15,14 +15,18 @@ inputs:
      required: false
      description: "Kubernetes that should be used"
      # renovate: datasource=github-releases depName=kubernetes/kubernetes
-     default: "v1.25.3"
+     default: "v1.26.0"
    functions_runtime_tag:
      description: "Tag for the functions runner image"
      required: true
    cluster-name:
      required: false
      description: "Name of the kind cluster"
      default: "test-cluster"
+   cluster-config:
+     required: false
+     description: "Config of the kind cluster"
+     default: "test/integration/kind-cluster-config.yaml"
 
 runs:
   using: "composite"
@@ -46,6 +50,7 @@ runs:
       version: ${{ inputs.kind-version }}
       node_image: "kindest/node:${{ inputs.k8s-version }}"
       kubectl_version: ${{ inputs.k8s-version }}
+      config: ${{ inputs.cluster-config }}
 
   - name: Import images in k3d
     shell: bash
@@ -71,6 +76,4 @@ runs:
       sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' ~/download/artifacts/keptn-lifecycle-operator-manifest-test/release.yaml
       sed -i 's/ghcr.keptn.sh\/keptn\/functions-runtime:.*/localhost:5000\/keptn\/functions-runtime:${{ inputs.functions_runtime_tag }}/g' ~/download/artifacts/keptn-lifecycle-operator-manifest-test/release.yaml
       kubectl apply -f ~/download/artifacts/keptn-lifecycle-operator-manifest-test
-      kubectl apply -f ~/download/artifacts/scheduler-manifest-test
-      kubectl rollout status deployment keptn-scheduler -n keptn-lifecycle-toolkit-system -w
       kubectl rollout status deployment klc-controller-manager -n keptn-lifecycle-toolkit-system -w

.github/workflows/CI.yaml

@@ -13,7 +13,6 @@ env:
   # renovate: datasource=github-releases depName=kubernetes-sigs/controller-tools
   CONTROLLER_TOOLS_VERSION: "v0.9.2"
   ENVTEST_K8S_VERSION: "1.24.2"
-  SCHEDULER_COMPATIBLE_K8S_VERSION: "v0.24.3"
 defaults:
   run:
     shell: bash
@@ -66,8 +65,6 @@ jobs:
         config:
           - name: "keptn-lifecycle-operator"
             folder: "operator/"
-          - name: "scheduler"
-            folder: "scheduler/"
           - name: "keptn-cert-manager"
             folder: "klt-cert-manager/"
     steps:
@@ -105,8 +102,6 @@ jobs:
         config:
           - name: "keptn-lifecycle-operator"
             folder: "operator/"
-          - name: "scheduler"
-            folder: "scheduler/"
           - name: "functions-runtime"
             folder: "functions-runtime/"
           - name: "keptn-cert-manager"
@@ -132,7 +127,6 @@ jobs:
             RELEASE_VERSION=dev-${{ env.DATETIME }}
             BUILD_TIME=${{ env.BUILD_TIME }}
             CONTROLLER_TOOLS_VERSION=${{ env.CONTROLLER_TOOLS_VERSION }}
-            SCHEDULER_COMPATIBLE_K8S_VERSION=${{ env.SCHEDULER_COMPATIBLE_K8S_VERSION }}
           builder: ${{ steps.buildx.outputs.name }}
           push: false
           cache-from: type=gha,scope=${{ github.ref_name }}-${{ matrix.config.name }}
@@ -204,8 +198,6 @@ jobs:
         config:
           - name: "keptn-lifecycle-operator"
             folder: "operator/"
-          - name: "scheduler"
-            folder: "scheduler/"
           - name: "functions-runtime"
             folder: "functions-runtime/"
           - name: "keptn-cert-manager"
@@ -240,7 +232,6 @@ jobs:
             RELEASE_VERSION=dev-${{ env.DATETIME }}
             BUILD_TIME=${{ env.BUILD_TIME }}
             CONTROLLER_TOOLS_VERSION=${{ env.CONTROLLER_TOOLS_VERSION }}
-            SCHEDULER_COMPATIBLE_K8S_VERSION=${{ env.SCHEDULER_COMPATIBLE_K8S_VERSION }}
           builder: ${{ steps.buildx.outputs.name }}
           push: true
           cache-from: type=gha,scope=${{ github.ref_name }}-${{ matrix.config.name }}

.github/workflows/e2e-test.yml

@@ -21,8 +21,6 @@ jobs:
         config:
           - name: "keptn-lifecycle-operator"
             folder: "operator/"
-          - name: "scheduler"
-            folder: "scheduler/"
     steps:
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/golangci-lint.yml

@@ -22,8 +22,6 @@ jobs:
         config:
           - name: "keptn-lifecycle-operator"
             folder: "operator/"
-          - name: "scheduler"
-            folder: "scheduler/"
     steps:
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/release.yml

@@ -15,7 +15,6 @@ env:
   GO_VERSION: "~1.19"
   # renovate: datasource=github-releases depName=kubernetes-sigs/controller-tools
   CONTROLLER_TOOLS_VERSION: "v0.9.2"
-  SCHEDULER_COMPATIBLE_K8S_VERSION: "v0.24.3"
 
 jobs:
   release-please:
@@ -45,8 +44,6 @@ jobs:
         config:
           - name: "keptn-lifecycle-operator"
             folder: "operator/"
-          - name: "scheduler"
-            folder: "scheduler/"
           - name: "functions-runtime"
             folder: "functions-runtime/"
     runs-on: ubuntu-22.04
@@ -89,7 +86,6 @@ jobs:
             RELEASE_VERSION=dev-${{ env.DATETIME }}
             BUILD_TIME=${{ env.BUILD_TIME }}
             CONTROLLER_TOOLS_VERSION=${{ env.CONTROLLER_TOOLS_VERSION }}
-            SCHEDULER_COMPATIBLE_K8S_VERSION=${{ env.SCHEDULER_COMPATIBLE_K8S_VERSION }}
           builder: ${{ steps.buildx.outputs.name }}
           push: true
           cache-from: type=gha,scope=${{ github.ref_name }}-${{ matrix.config.name }}
@@ -133,13 +129,11 @@ jobs:
           RELEASE_REGISTRY: ghcr.keptn.sh/keptn
           TAG: ${{ needs.release-please.outputs.tag_name }}
         run: |
-          cd scheduler
-          make release-manifests
           cd ../operator
           make controller-gen release-manifests
           cd ..
           echo "---" >> operator/config/rendered/release.yaml
-          cat operator/config/rendered/release.yaml scheduler/config/rendered/release.yaml > manifest.yaml
+          cat operator/config/rendered/release.yaml > manifest.yaml
 
       - name: Attach release assets
         uses: softprops/action-gh-release@v1

.github/workflows/security-scans.yml

@@ -49,7 +49,6 @@ jobs:
           name: manifests
           path: |
             ./dist/keptn-lifecycle-operator-manifest/
-            ./dist/scheduler-manifest/
 
       - name: Upload images
         uses: actions/upload-artifact@v3
@@ -58,7 +57,6 @@ jobs:
           path: |
             ./dist/functions-runtime-image.tar/
             ./dist/keptn-lifecycle-operator-image.tar/
-            ./dist/scheduler-image.tar/
 
   security-scans:
     name: "Security Scans"
@@ -124,12 +122,11 @@ jobs:
           cd operator/config/default
           sed -i 's/\- ..\/crd//' kustomization.yaml && kustomize build ./ > /tmp/operator-manifest.yaml
           cd ../crd && kustomize build ./ > /tmp/crds.yaml
-          cd ../../../scheduler/manifests/install && kustomize build ./ > /tmp/scheduler-manifest.yaml
 
           curl -s https://raw.githubusercontent.com/yannh/kubeconform/${{ matrix.version }}/scripts/openapi2jsonschema.py > /tmp/openapi2jsonschema.py
           mkdir -p /tmp/schema && cd /tmp/schema
           python3 ../openapi2jsonschema.py ../crds.yaml
-          cd .. && echo "---" >> operator-manifest.yaml && cat operator-manifest.yaml scheduler-manifest.yaml > manifest.yaml
+          cd .. && echo "---" >> operator-manifest.yaml && cat operator-manifest.yaml > manifest.yaml
           kubeconform -schema-location default -schema-location './schema/{{ .ResourceKind }}_{{ .ResourceAPIVersion }}.json' -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cert-manager.io/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -summary manifest.yaml
 
       - name: Kubescape Scan
@@ -153,7 +150,6 @@ jobs:
         image:
           - "functions-runtime"
           - "keptn-lifecycle-operator"
-          - "scheduler"
     steps:
       - name: Download images
         id: download_images
@@ -177,7 +173,6 @@ jobs:
       matrix:
         artifact:
           - "operator"
-          - "scheduler"
 
     steps:
       - name: Set up Go 1.x

.github/workflows/validate-semantic-pr.yml

@@ -33,7 +33,6 @@ jobs:
             test
             deps
           scopes: |
-            scheduler
             operator
             functions-runtime
             dashboards

Makefile

@@ -30,18 +30,10 @@ build-deploy-operator: deploy-cert-manager
 
 	kubectl apply -f operator/config/rendered/release.yaml
 
-.PHONY: build-deploy-scheduler
-build-deploy-scheduler:
-	$(MAKE) -C scheduler release-local.$(ARCH) RELEASE_REGISTRY=$(RELEASE_REGISTRY) TAG=$(TAG)
-	$(MAKE) -C scheduler push-local RELEASE_REGISTRY=$(RELEASE_REGISTRY) TAG=$(TAG)
-	$(MAKE) -C scheduler release-manifests RELEASE_REGISTRY=$(RELEASE_REGISTRY) TAG=$(TAG) ARCH=$(ARCH)
-	kubectl create namespace keptn-lifecycle-toolkit-system --dry-run=client -o yaml | kubectl apply -f -
-	kubectl apply -f scheduler/config/rendered/release.yaml
-
 .PHONY: deploy-cert-manager
 deploy-cert-manager:
 	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
 
 .PHONY: build-deploy-dev-environment
-build-deploy-dev-environment: build-deploy-operator build-deploy-scheduler
+build-deploy-dev-environment: build-deploy-operator
 	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml

codecov.yml

@@ -28,10 +28,6 @@ flags:
     paths:
       - operator/
     carryforward: true
-  scheduler:
-    paths:
-      - scheduler/
-    carryforward: true
   component-tests:
     paths:
       - operator/

examples/observability/pod.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+spec:
+  containers:
+  - name: nginx
+    image: nginx:1.14.2
+    ports:
+    - containerPort: 80

examples/support/observability/Makefile

@@ -125,7 +125,7 @@ deploy-podtatohead-failed-pre:
 	kubectl create namespace "$(PODTATO_NAMESPACE)" --dry-run=client -o yaml | kubectl apply -f -
 	kubectl annotate namespace "$(PODTATO_NAMESPACE)" keptn.sh/lifecycle-toolkit=enabled --overwrite
 	kubectl apply -f ./assets/podtatohead-deployment-evaluation/.
-	kubectl apply -f ./assets/podtatohead-evaluationdefinitions/keptnevaluationdefinition3.yaml
+	kubectl apply -f ./assets/podtatohead-evaluationdefinitions/keptnevaluationdefinition2.yaml
 	kubectl apply -f ./assets/podtatohead-evaluationdefinitions/keptnevaluationdefinition4.yaml
 	kubectl wait --for=condition=available deployment/podtato-head-entry -n podtato-kubectl --timeout=120s
 	@echo "Watch Workload Deployment Progress: kubectl get keptnworkloadinstances -n podtato-kubectl -w"

examples/support/observability/assets/podtatohead-deployment-evaluation/app.yaml

@@ -21,6 +21,6 @@ spec:
   postDeploymentTasks:
     - post-deployment-hello
   preDeploymentEvaluations:     #Comment out this four lines if you do not have a prometheus installation
-    - my-prometheus-definition1
+    - my-prometheus-definition2
   postDeploymentEvaluations:
     - my-prometheus-definition2

examples/support/observability/assets/podtatohead-deployment-evaluation/manifest.yaml

@@ -30,7 +30,7 @@ spec:
       initContainers:
       - name: init-myservice
         image: busybox:1.28
-        command: ['sh', '-c', 'sleep 30']
+        command: ['sh', '-c', 'sleep 10']
       containers:
       - name: server
         image: ghcr.io/podtato-head/entry:latest
@@ -77,7 +77,7 @@ spec:
         component: podtato-head-hat
         keptn.sh/pre-deployment-tasks: check-entry-service
         keptn.sh/pre-deployment-evaluations: my-prometheus-definition1
-        keptn.sh/post-deployment-evaluations: my-prometheus-definition1
+        keptn.sh/post-deployment-evaluations: my-prometheus-definition2
       annotations:
         keptn.sh/app: podtato-head
         keptn.sh/workload: podtato-head-hat

operator/apis/lifecycle/v1alpha2/common/common.go

@@ -19,6 +19,7 @@ const K8sRecommendedVersionAnnotations = "app.kubernetes.io/version"
 const K8sRecommendedAppAnnotations = "app.kubernetes.io/part-of"
 const PreDeploymentEvaluationAnnotation = "keptn.sh/pre-deployment-evaluations"
 const PostDeploymentEvaluationAnnotation = "keptn.sh/post-deployment-evaluations"
+const SchedullingGateRemoved = "keptn.sh/schedulling-gate-removed"
 const TaskNameAnnotation = "keptn.sh/task-name"
 const NamespaceEnabledAnnotation = "keptn.sh/lifecycle-toolkit"
 const CreateAppTaskSpanName = "create_%s_app_task"

operator/config/rbac/role.yaml

@@ -66,6 +66,7 @@ rules:
   verbs:
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - ""

operator/controllers/common/helperfunctions.go

@@ -1,9 +1,15 @@
 package common
 
 import (
+	"context"
+
+	"github.com/go-logr/logr"
 	klcv1alpha2 "github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha2"
 	apicommon "github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha2/common"
+	controllererrors "github.com/keptn/lifecycle-toolkit/operator/controllers/errors"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type CreateAttributes struct {
@@ -41,3 +47,64 @@ func GetOldStatus(name string, statuses []klcv1alpha2.ItemStatus) apicommon.Kept
 
 	return oldstatus
 }
+
+func RemoveGates(ctx context.Context, c client.Client, log logr.Logger, workloadInstance *klcv1alpha2.KeptnWorkloadInstance) error {
+	switch workloadInstance.Spec.ResourceReference.Kind {
+	case "Pod":
+		return removePodGates(ctx, c, log, workloadInstance.Spec.ResourceReference.Name, workloadInstance.Namespace)
+	case "ReplicaSet", "StatefulSet", "DaemonSet":
+		podList, err := getPodsOfOwner(ctx, c, log, workloadInstance.Spec.ResourceReference.UID, workloadInstance.Spec.ResourceReference.Kind, workloadInstance.Namespace)
+		if err != nil {
+			log.Error(err, "cannot get pods")
+			return err
+		}
+		for _, pod := range podList {
+			err := removePodGates(ctx, c, log, pod, workloadInstance.Namespace)
+			if err != nil {
+				log.Error(err, "cannot remove gates from pod")
+				return err
+			}
+		}
+	default:
+		return controllererrors.ErrUnsupportedWorkloadInstanceResourceReference
+	}
+
+	return nil
+}
+
+func removePodGates(ctx context.Context, c client.Client, log logr.Logger, podName string, podNamespace string) error {
+	pod := &v1.Pod{}
+	err := c.Get(ctx, types.NamespacedName{Namespace: podNamespace, Name: podName}, pod)
+	if err != nil {
+		log.Error(err, "cannot remove gates from pod - inner")
+		return err
+	}
+	if len(pod.Annotations) == 0 {
+		pod.Annotations = make(map[string]string)
+	}
+	pod.Annotations[apicommon.SchedullingGateRemoved] = "true"
+	pod.Spec.SchedulingGates = nil
+	return c.Update(ctx, pod)
+}
+
+func getPodsOfOwner(ctx context.Context, c client.Client, log logr.Logger, ownerUID types.UID, ownerKind string, namespace string) ([]string, error) {
+	pods := &v1.PodList{}
+	err := c.List(ctx, pods, client.InNamespace(namespace))
+	if err != nil {
+		log.Error(err, "cannot list pods - inner")
+		return nil, err
+	}
+
+	var resultPods []string
+
+	for _, pod := range pods.Items {
+		for _, owner := range pod.OwnerReferences {
+			if owner.Kind == ownerKind && owner.UID == ownerUID {
+				resultPods = append(resultPods, pod.Name)
+				break
+			}
+		}
+	}
+
+	return resultPods, nil
+}