Configurable cron schedule for reconciler

.github/workflows/build.yml

@@ -4,7 +4,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         #goarch: [386, amd64, arm, ppc64le, arm64]
         goarch: [amd64, arm64]
         os: [ubuntu-latest] #, macos-latest, windows-latest]

.github/workflows/test.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -74,7 +74,7 @@ jobs:
       - name: Set up Go version
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
 
       - name: Checkout code into the Go module directory
         uses: actions/checkout@v2

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19
+FROM golang:1.20
 ADD . /usr/src/whereabouts
 RUN mkdir -p $GOPATH/src/github.com/k8snetworkplumbingwg/whereabouts
 WORKDIR $GOPATH/src/github.com/k8snetworkplumbingwg/whereabouts

Dockerfile.arm64

@@ -1,4 +1,4 @@
-FROM golang:1.19
+FROM golang:1.20
 ADD . /usr/src/whereabouts
 
 ENV GOARCH "arm64"

Dockerfile.openshift

@@ -1,5 +1,5 @@
 # This dockerfile is used for building for OpenShift
-FROM registry.ci.openshift.org/openshift/release:golang-1.19 as builder
+FROM registry.ci.openshift.org/openshift/release:golang-1.20 as builder
 ADD . /go/src/github.com/k8snetworkplumbingwg/whereabouts
 WORKDIR /go/src/github.com/k8snetworkplumbingwg/whereabouts
 ENV CGO_ENABLED=1

cmd/controlloop/controlloop.go

@@ -7,7 +7,8 @@ import (
 	"os/signal"
 	"time"
 
-	gocron "github.com/go-co-op/gocron"
+	"github.com/fsnotify/fsnotify"
+	"github.com/go-co-op/gocron/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -20,23 +21,23 @@ import (
 
 	wbclient "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/clientset/versioned"
 	wbinformers "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/informers/externalversions"
-	"github.com/k8snetworkplumbingwg/whereabouts/pkg/config"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/controlloop"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/logging"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler"
-	"github.com/k8snetworkplumbingwg/whereabouts/pkg/types"
 )
 
 const (
-	allNamespaces  = ""
-	controllerName = "pod-ip-controlloop"
+	allNamespaces               = ""
+	controllerName              = "pod-ip-controlloop"
+	reconcilerCronConfiguration = "/cron-schedule/config"
 )
 
 const (
-	couldNotCreateController = 1
-	couldNotReadFlatfile     = 1
-	couldNotGetFlatIPAM      = 1
-	cronExpressionError      = 1
+	_ int = iota
+	couldNotCreateController
+	cronSchedulerCreationError
+	fileWatcherError
+	couldNotCreateConfigWatcherError
 )
 
 const (
@@ -65,24 +66,44 @@ func main() {
 	networkController.Start(stopChan)
 	defer networkController.Shutdown()
 
-	s := gocron.NewScheduler(time.UTC)
-	schedule := cronExpressionFromFlatFile()
+	s, err := gocron.NewScheduler(gocron.WithLocation(time.UTC))
+	if err != nil {
+		os.Exit(cronSchedulerCreationError)
+	}
 
-	_, err = s.Cron(schedule).Do(func() { // user configurable cron expression in install-cni.sh
-		reconciler.ReconcileIPs(errorChan)
-	})
+	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
-		_ = logging.Errorf("error with cron expression schedule: %v", err)
-		os.Exit(cronExpressionError)
+		_ = logging.Errorf("error creating configuration watcher: %v", err)
+		os.Exit(fileWatcherError)
 	}
+	defer watcher.Close()
+
+	reconcilerConfigWatcher, err := reconciler.NewConfigWatcher(
+		reconcilerCronConfiguration,
+		s,
+		watcher,
+		func() {
+			reconciler.ReconcileIPs(errorChan)
+		},
+	)
+	if err != nil {
+		os.Exit(couldNotCreateConfigWatcherError)
+	}
+	s.Start()
 
-	s.StartAsync()
+	const reconcilerConfigMntFile = "/cron-schedule/..data"
+	p := func(e fsnotify.Event) bool {
+		return e.Name == reconcilerConfigMntFile && e.Op&fsnotify.Create == fsnotify.Create
+	}
+	reconcilerConfigWatcher.SyncConfiguration(p)
 
 	for {
 		select {
 		case <-stopChan:
 			logging.Verbosef("shutting down network controller")
-			s.Stop()
+			if err := s.Shutdown(); err != nil {
+				_ = logging.Errorf("error shutting : %v", err)
+			}
 			return
 		case err := <-errorChan:
 			if err == nil {
@@ -163,12 +184,3 @@ func newEventBroadcaster(k8sClientset kubernetes.Interface) record.EventBroadcas
 func newEventRecorder(broadcaster record.EventBroadcaster) record.EventRecorder {
 	return broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerName})
 }
-
-func cronExpressionFromFlatFile() string {
-	flatipam, _, err := config.GetFlatIPAM(true, &types.IPAMConfig{}, "")
-	if err != nil {
-		_ = logging.Errorf("could not get flatipam: %v", err)
-		os.Exit(couldNotGetFlatIPAM)
-	}
-	return flatipam.IPAM.ReconcilerCronExpression
-}

doc/crds/daemonset-install.yaml

@@ -70,6 +70,18 @@ rules:
   - patch
   - update
   - get
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: whereabouts-config
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      Configmap containing user customizable cronjob schedule
+data:
+  cron-expression: "30 4 * * *" # Default schedule is once per day at 4:30am. Users may configure this value to their liking.
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -130,10 +142,19 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: cni-net-dir
           mountPath: /host/etc/cni/net.d
+        - name: cron-scheduler-configmap
+          mountPath: /cron-schedule
       volumes:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
         - name: cni-net-dir
           hostPath:
             path: /etc/cni/net.d
+        - name: cron-scheduler-configmap
+          configMap:
+            name: "whereabouts-config"
+            defaultMode: 0744
+            items:
+            - key: "cron-expression"
+              path: "config"

doc/extended-configuration.md

@@ -134,16 +134,29 @@ spec:
 
 You'll note that in the `ipam` section there's a lot less parameters than are used in the previous examples.
 
-### Reconciler Cron Expression Configuration (optional)
+## Reconciler Cron Expression configuration for clusters via flatfile (optional)
 
-You may want to provide a cron expression to configure how frequently the ip-reconciler runs. This is done via the flatfile.
+*NOTE: configuring cron expression prior to cluster launch will only work for **non-Openshift** Kubernetes clusters, such as a vanilla Kubernetes cluster. Skip to the next section if you have an Openshift cluster or a cluster that has already launched.*
+
+You may want to provide a cron expression to configure how frequently the ip-reconciler runs. For clusters that have not yet been launched, this can be configured via the flatfile.
 
 You can speficy the `WHEREABOUTS_RECONCILER_CRON` environment variable in your daemonset definition file to override the default cron expression:
 ```yaml
        env:
         - name: WHEREABOUTS_RECONCILER_CRON
           value: 30 * * * *
 ```
+
+## Reconciler Cron Expression Configuration for live clusters via configmap (optional)
+
+You may want to provide a cron expression to configure how frequently the ip-reconciler runs. For **Openshift** Kubernetes clusters, this is done via updating the cron-scheduler-configmap. 
+
+You can check that the cron-scheduler-configmap is present by running `oc get configmaps` in the openshift-multus namespace.
+
+To update the cron-scheduler-configmap, run `oc edit configmap cron-scheduler-configmap` and adjust the value to a valid cron expression of your liking. Shortly after, the reconciler schedule will update.
+
+If you are using a non-Openshift cluster, you can do the same steps, but you will need to look for the configmap in the kube-system namespace.
+
 ## Installing etcd. (optional)
 
 etcd installation is optional. By default, we recommend the custom resource backend (given in the first example configuration).

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.8.2
-	github.com/go-co-op/gocron v1.13.0
 	github.com/imdario/mergo v0.3.12
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05
@@ -21,17 +20,20 @@ require (
 	k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6
 )
 
+require github.com/go-co-op/gocron/v2 v2.1.0
+
 require (
+	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
 	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
@@ -42,22 +44,22 @@ require (
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect