From fd5223801a5a1d2d32dd26de548ea4df8b9f2f0e Mon Sep 17 00:00:00 2001 From: Luke Addison Date: Fri, 20 Dec 2024 16:33:32 +0000 Subject: [PATCH 1/3] Add scheduled Pod limiter --- README.md | 1 + go.mod | 4 ++ go.sum | 4 +- pkg/api/v1alpha1/types.go | 12 ++++ pkg/config/config_test.go | 13 ++++ pkg/controller/scheduled_pod_limiter.go | 61 +++++++++++++++++++ pkg/controller/scheduled_pod_limiter_test.go | 64 ++++++++++++++++++++ 7 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 pkg/controller/scheduled_pod_limiter.go create mode 100644 pkg/controller/scheduled_pod_limiter_test.go diff --git a/README.md b/README.md index bb024c1..cfe1119 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ config: controllers: - spot-migrator - pod-safe-to-evict-annotator + - scheduled-pod-limiter cloudProvider: name: gcp podSafeToEvictAnnotator: diff --git a/go.mod b/go.mod index 3d4cadf..4251b0d 100644 --- a/go.mod +++ b/go.mod @@ -146,3 +146,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +// Replace cron library with fork containing Prev function: https://github.com/robfig/cron/pull/437 +// go mod edit -replace github.com/robfig/cron/v3=github.com/juliev0/cron/v3@7181f74c09e99418674d5ef1e438066a4b841a7a +replace github.com/robfig/cron/v3 => github.com/juliev0/cron/v3 v3.0.2-0.20220310063235-7181f74c09e9 diff --git a/go.sum b/go.sum index dcde14d..389bf64 100644 --- a/go.sum +++ b/go.sum @@ -173,6 +173,8 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juliev0/cron/v3 v3.0.2-0.20220310063235-7181f74c09e9 h1:Pa9SanmckPLZ4HeHEl/OGxXPKRg5NpNx3xmM8wDN/LE= +github.com/juliev0/cron/v3 v3.0.2-0.20220310063235-7181f74c09e9/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -235,8 +237,6 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= diff --git a/pkg/api/v1alpha1/types.go b/pkg/api/v1alpha1/types.go index fe5b7b0..2565493 100644 --- a/pkg/api/v1alpha1/types.go +++ b/pkg/api/v1alpha1/types.go @@ -13,6 +13,7 @@ type CostManagerConfiguration struct { CloudProvider CloudProvider `json:"cloudProvider"` SpotMigrator *SpotMigrator `json:"spotMigrator,omitempty"` PodSafeToEvictAnnotator *PodSafeToEvictAnnotator `json:"podSafeToEvictAnnotator,omitempty"` + ScheduledPodLimiter *ScheduledPodLimiter `json:"scheduledPodLimiter,omitempty"` } type CloudProvider struct { @@ -26,3 +27,14 @@ type SpotMigrator struct { type PodSafeToEvictAnnotator struct { NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` } + +type ScheduledPodLimiter struct { + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` + SchedulePolicy *SchedulePolicy `json:"schedulePolicy,omitempty"` +} + +type SchedulePolicy struct { + StartSchedule string `json:"startSchedule,omitempty"` + StopSchedule string `json:"stopSchedule,omitempty"` + TimeZone string `json:"timeZone,omitempty"` +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 17833a5..9b87f0c 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -22,6 +22,7 @@ kind: CostManagerConfiguration controllers: - spot-migrator - pod-safe-to-evict-annotator +- scheduled-pod-limiter cloudProvider: name: gcp spotMigrator: @@ -33,6 +34,18 @@ podSafeToEvictAnnotator: operator: In values: - kube-system +scheduledPodLimiter: + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - cost-manager + schedulePolicy: + startSchedule: "0 8 * * Mon-Fri" + stopSchedule: "0 18 * * Mon-Fri" + timeZone: "Europe/London" `), valid: true, config: &v1alpha1.CostManagerConfiguration{ diff --git a/pkg/controller/scheduled_pod_limiter.go b/pkg/controller/scheduled_pod_limiter.go new file mode 100644 index 0000000..635c129 --- /dev/null +++ b/pkg/controller/scheduled_pod_limiter.go @@ -0,0 +1,61 @@ +package controller + +import ( + "context" + "time" + + "github.com/hsbc/cost-manager/pkg/api/v1alpha1" + "github.com/robfig/cron/v3" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + scheduledPodLimiterControllerName = "scheduled-pod-limiter" +) + +// scheduledPodLimiter creates ResourceQuotas in selected Namespaces at the scheduled stop times to +// limit Pods to 0 and then deletes all Pods; this should trigger the cluster autoscaler to scale +// the cluster down. At the scheduled start times the ResourceQuotas are deleted, allowing the Pods +// to be recreated +type scheduledPodLimiter struct { + Config *v1alpha1.ScheduledPodLimiter + Client client.Client + startSchedule cron.Schedule + stopSchedule cron.Schedule +} + +var _ reconcile.Reconciler = &scheduledPodLimiter{} + +func (s *scheduledPodLimiter) SetupWithManager(mgr ctrl.Manager) error { + // Parse cron schedules + startSchedule, err := cron.ParseStandard(s.Config.SchedulePolicy.StartSchedule) + if err != nil { + return err + } + s.startSchedule = startSchedule + stopSchedule, err := cron.ParseStandard(s.Config.SchedulePolicy.StopSchedule) + if err != nil { + return err + } + s.stopSchedule = stopSchedule + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + Complete(s) +} + +func (s *scheduledPodLimiter) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + // TODO: Determine whether we should limit workloads + + return reconcile.Result{}, nil +} + +func (s *scheduledPodLimiter) shouldLimitPods(now time.Time) bool { + previousStartTime := s.startSchedule.Prev(now) + previousStopTime := s.stopSchedule.Prev(now) + + return previousStopTime.After(previousStartTime) +} diff --git a/pkg/controller/scheduled_pod_limiter_test.go b/pkg/controller/scheduled_pod_limiter_test.go new file mode 100644 index 0000000..40aa76e --- /dev/null +++ b/pkg/controller/scheduled_pod_limiter_test.go @@ -0,0 +1,64 @@ +package controller + +import ( + "testing" + "time" + + "github.com/robfig/cron/v3" + "github.com/stretchr/testify/require" +) + +func TestShouldLimitPods(t *testing.T) { + tests := map[string]struct { + startSchedule string + stopSchedule string + time time.Time + shouldLimitPods bool + }{ + "startAndStopEveryMinute": { + startSchedule: "* * * * *", + stopSchedule: "* * * * *", + time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + shouldLimitPods: false, + }, + "startEvenMinutesStopOddMinutesAt30SecondsPastEvenMinute": { + startSchedule: "*/2 * * * *", + stopSchedule: "1-59/2 * * * *", + time: time.Date(2024, 1, 1, 0, 0, 30, 0, time.UTC), + shouldLimitPods: false, + }, + "startEvenMinutesStopOddMinutesAt30SecondsPastOddMinute": { + startSchedule: "*/2 * * * *", + stopSchedule: "1-59/2 * * * *", + time: time.Date(2024, 1, 1, 0, 1, 30, 0, time.UTC), + shouldLimitPods: true, + }, + "startMondayMorningStopFridayEveningDuringWeek": { + startSchedule: "0 8 * * Mon", + stopSchedule: "0 16 * * Fri", + // Wednesday, January 10, 2024 8:00:00 AM + time: time.Date(2024, 1, 10, 8, 0, 0, 0, time.UTC), + shouldLimitPods: false, + }, + "startMondayMorningStopFridayEveningDuringWeekend": { + startSchedule: "0 8 * * Mon", + stopSchedule: "0 16 * * Fri", + // Sunday, January 14, 2024 10:00:00 PM + time: time.Date(2024, 1, 14, 22, 0, 0, 0, time.UTC), + shouldLimitPods: true, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + startSchedule, err := cron.ParseStandard(test.startSchedule) + require.Nil(t, err) + stopSchedule, err := cron.ParseStandard(test.stopSchedule) + require.Nil(t, err) + s := &scheduledPodLimiter{ + startSchedule: startSchedule, + stopSchedule: stopSchedule, + } + require.Equal(t, test.shouldLimitPods, s.shouldLimitPods(test.time)) + }) + } +} From 71ec8d37e4800ba413c2e9861080d248499ede0e Mon Sep 17 00:00:00 2001 From: Luke Addison Date: Sat, 21 Dec 2024 22:31:16 +0000 Subject: [PATCH 2/3] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index bd7df34..e3f4661 100644 --- a/go.sum +++ b/go.sum @@ -199,8 +199,6 @@ github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFS github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= From 5e228298346dbd434de95f8143a4bc1022773ca7 Mon Sep 17 00:00:00 2001 From: Luke Addison Date: Sat, 21 Dec 2024 22:41:46 +0000 Subject: [PATCH 3/3] Generate DeepCopy --- pkg/api/v1alpha1/zz_generated.deepcopy.go | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 33b2e30..4a39f5b 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -44,6 +44,11 @@ func (in *CostManagerConfiguration) DeepCopyInto(out *CostManagerConfiguration) *out = new(PodSafeToEvictAnnotator) (*in).DeepCopyInto(*out) } + if in.ScheduledPodLimiter != nil { + in, out := &in.ScheduledPodLimiter, &out.ScheduledPodLimiter + *out = new(ScheduledPodLimiter) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CostManagerConfiguration. @@ -84,6 +89,46 @@ func (in *PodSafeToEvictAnnotator) DeepCopy() *PodSafeToEvictAnnotator { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulePolicy) DeepCopyInto(out *SchedulePolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulePolicy. +func (in *SchedulePolicy) DeepCopy() *SchedulePolicy { + if in == nil { + return nil + } + out := new(SchedulePolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScheduledPodLimiter) DeepCopyInto(out *ScheduledPodLimiter) { + *out = *in + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.SchedulePolicy != nil { + in, out := &in.SchedulePolicy, &out.SchedulePolicy + *out = new(SchedulePolicy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledPodLimiter. +func (in *ScheduledPodLimiter) DeepCopy() *ScheduledPodLimiter { + if in == nil { + return nil + } + out := new(ScheduledPodLimiter) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SpotMigrator) DeepCopyInto(out *SpotMigrator) { *out = *in