✨ Add scheduled Pod limiter

README.md

@@ -96,6 +96,7 @@ config:
   controllers:
   - spot-migrator
   - pod-safe-to-evict-annotator
+  - scheduled-pod-limiter
   cloudProvider:
     name: gcp
   podSafeToEvictAnnotator:

go.mod

@@ -150,3 +150,7 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+// Replace cron library with fork containing Prev function: https://github.com/robfig/cron/pull/437
+// go mod edit -replace github.com/robfig/cron/v3=github.com/juliev0/cron/v3@7181f74c09e99418674d5ef1e438066a4b841a7a
+replace github.com/robfig/cron/v3 => github.com/juliev0/cron/v3 v3.0.2-0.20220310063235-7181f74c09e9

go.sum

@@ -134,6 +134,8 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/juliev0/cron/v3 v3.0.2-0.20220310063235-7181f74c09e9 h1:Pa9SanmckPLZ4HeHEl/OGxXPKRg5NpNx3xmM8wDN/LE=
+github.com/juliev0/cron/v3 v3.0.2-0.20220310063235-7181f74c09e9/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
@@ -197,8 +199,6 @@ github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFS
 github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
-github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=

pkg/api/v1alpha1/types.go

@@ -13,6 +13,7 @@ type CostManagerConfiguration struct {
 	CloudProvider           CloudProvider            `json:"cloudProvider"`
 	SpotMigrator            *SpotMigrator            `json:"spotMigrator,omitempty"`
 	PodSafeToEvictAnnotator *PodSafeToEvictAnnotator `json:"podSafeToEvictAnnotator,omitempty"`
+	ScheduledPodLimiter     *ScheduledPodLimiter     `json:"scheduledPodLimiter,omitempty"`
 }
 
 type CloudProvider struct {
@@ -26,3 +27,14 @@ type SpotMigrator struct {
 type PodSafeToEvictAnnotator struct {
 	NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"`
 }
+
+type ScheduledPodLimiter struct {
+	NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"`
+	SchedulePolicy    *SchedulePolicy       `json:"schedulePolicy,omitempty"`
+}
+
+type SchedulePolicy struct {
+	StartSchedule string `json:"startSchedule,omitempty"`
+	StopSchedule  string `json:"stopSchedule,omitempty"`
+	TimeZone      string `json:"timeZone,omitempty"`
+}

pkg/config/config_test.go

@@ -22,6 +22,7 @@ kind: CostManagerConfiguration
 controllers:
 - spot-migrator
 - pod-safe-to-evict-annotator
+- scheduled-pod-limiter
 cloudProvider:
   name: gcp
 spotMigrator:
@@ -33,6 +34,18 @@ podSafeToEvictAnnotator:
       operator: In
       values:
       - kube-system
+scheduledPodLimiter:
+  namespaceSelector:
+    matchExpressions:
+	- key: kubernetes.io/metadata.name
+	  operator: NotIn
+	  values:
+	  - kube-system
+	  - cost-manager
+  schedulePolicy:
+    startSchedule: "0 8 * * Mon-Fri"
+	stopSchedule: "0 18 * * Mon-Fri"
+	timeZone: "Europe/London"
 `),
 			valid: true,
 			config: &v1alpha1.CostManagerConfiguration{

pkg/controller/scheduled_pod_limiter.go

@@ -0,0 +1,61 @@
+package controller
+
+import (
+	"context"
+	"time"
+
+	"github.com/hsbc/cost-manager/pkg/api/v1alpha1"
+	"github.com/robfig/cron/v3"
+	corev1 "k8s.io/api/core/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	scheduledPodLimiterControllerName = "scheduled-pod-limiter"
+)
+
+// scheduledPodLimiter creates ResourceQuotas in selected Namespaces at the scheduled stop times to
+// limit Pods to 0 and then deletes all Pods; this should trigger the cluster autoscaler to scale
+// the cluster down. At the scheduled start times the ResourceQuotas are deleted, allowing the Pods
+// to be recreated
+type scheduledPodLimiter struct {
+	Config        *v1alpha1.ScheduledPodLimiter
+	Client        client.Client
+	startSchedule cron.Schedule
+	stopSchedule  cron.Schedule
+}
+
+var _ reconcile.Reconciler = &scheduledPodLimiter{}
+
+func (s *scheduledPodLimiter) SetupWithManager(mgr ctrl.Manager) error {
+	// Parse cron schedules
+	startSchedule, err := cron.ParseStandard(s.Config.SchedulePolicy.StartSchedule)
+	if err != nil {
+		return err
+	}
+	s.startSchedule = startSchedule
+	stopSchedule, err := cron.ParseStandard(s.Config.SchedulePolicy.StopSchedule)
+	if err != nil {
+		return err
+	}
+	s.stopSchedule = stopSchedule
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Pod{}).
+		Complete(s)
+}
+
+func (s *scheduledPodLimiter) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	// TODO: Determine whether we should limit workloads
+
+	return reconcile.Result{}, nil
+}
+
+func (s *scheduledPodLimiter) shouldLimitPods(now time.Time) bool {
+	previousStartTime := s.startSchedule.Prev(now)
+	previousStopTime := s.stopSchedule.Prev(now)
+
+	return previousStopTime.After(previousStartTime)
+}

pkg/controller/scheduled_pod_limiter_test.go

@@ -0,0 +1,64 @@
+package controller
+
+import (
+	"testing"
+	"time"
+
+	"github.com/robfig/cron/v3"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShouldLimitPods(t *testing.T) {
+	tests := map[string]struct {
+		startSchedule   string
+		stopSchedule    string
+		time            time.Time
+		shouldLimitPods bool
+	}{
+		"startAndStopEveryMinute": {
+			startSchedule:   "* * * * *",
+			stopSchedule:    "* * * * *",
+			time:            time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
+			shouldLimitPods: false,
+		},
+		"startEvenMinutesStopOddMinutesAt30SecondsPastEvenMinute": {
+			startSchedule:   "*/2 * * * *",
+			stopSchedule:    "1-59/2 * * * *",
+			time:            time.Date(2024, 1, 1, 0, 0, 30, 0, time.UTC),
+			shouldLimitPods: false,
+		},
+		"startEvenMinutesStopOddMinutesAt30SecondsPastOddMinute": {
+			startSchedule:   "*/2 * * * *",
+			stopSchedule:    "1-59/2 * * * *",
+			time:            time.Date(2024, 1, 1, 0, 1, 30, 0, time.UTC),
+			shouldLimitPods: true,
+		},
+		"startMondayMorningStopFridayEveningDuringWeek": {
+			startSchedule: "0 8 * * Mon",
+			stopSchedule:  "0 16 * * Fri",
+			// Wednesday, January 10, 2024 8:00:00 AM
+			time:            time.Date(2024, 1, 10, 8, 0, 0, 0, time.UTC),
+			shouldLimitPods: false,
+		},
+		"startMondayMorningStopFridayEveningDuringWeekend": {
+			startSchedule: "0 8 * * Mon",
+			stopSchedule:  "0 16 * * Fri",
+			// Sunday, January 14, 2024 10:00:00 PM
+			time:            time.Date(2024, 1, 14, 22, 0, 0, 0, time.UTC),
+			shouldLimitPods: true,
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			startSchedule, err := cron.ParseStandard(test.startSchedule)
+			require.Nil(t, err)
+			stopSchedule, err := cron.ParseStandard(test.stopSchedule)
+			require.Nil(t, err)
+			s := &scheduledPodLimiter{
+				startSchedule: startSchedule,
+				stopSchedule:  stopSchedule,
+			}
+			require.Equal(t, test.shouldLimitPods, s.shouldLimitPods(test.time))
+		})
+	}
+}