WIP: Teach the CVO the new configuration API

cmd/updatestatuscontroller.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"k8s.io/utils/clock"
 
 	k8sversion "k8s.io/apimachinery/pkg/version"
 
@@ -19,6 +20,7 @@ func init() {
 		// https://github.com/openshift/cluster-version-operator/pull/1091#discussion_r1810601697
 		k8sversion.Info{GitVersion: cvoversion.Raw},
 		updatestatus.Run,
+		clock.RealClock{},
 	).NewCommandWithContext(context.Background())
 
 	uscCommand.Short = "The Update Status Controller watches cluster state/health during the update process and exposes it through the UpdateStatus API."

go.mod

@@ -11,8 +11,8 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/openshift/api v0.0.0-20241216151652-de9de05a8e43
-	github.com/openshift/client-go v0.0.0-20241107164952-923091dd2b1a
-	github.com/openshift/library-go v0.0.0-20241107160307-0064ad7bd060
+	github.com/openshift/client-go v0.0.0-20241217083110-35abaf51555b
+	github.com/openshift/library-go v0.0.0-20241218091209-6018a90c28d0
 	github.com/operator-framework/api v0.17.1
 	github.com/operator-framework/operator-lifecycle-manager v0.22.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -148,10 +148,10 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
 github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
 github.com/openshift/api v0.0.0-20241216151652-de9de05a8e43 h1:3lcB5nqOOfsJzY4JD12AMKyg3+yQhAdJzNDenbmbMQg=
 github.com/openshift/api v0.0.0-20241216151652-de9de05a8e43/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
-github.com/openshift/client-go v0.0.0-20241107164952-923091dd2b1a h1:h3F55x+zOXwSG4okUlXmOlU/5CQ7Ve2HKxYjseJPVoY=
-github.com/openshift/client-go v0.0.0-20241107164952-923091dd2b1a/go.mod h1:JBIcn1JfD/JCpGYnRRT0+HLxrleF/Y7T3Y/t0p8o5jk=
-github.com/openshift/library-go v0.0.0-20241107160307-0064ad7bd060 h1:jiDC7d8d+jmjv2WfiMY0+Uf55q11MGyYkGGqXnfqWTU=
-github.com/openshift/library-go v0.0.0-20241107160307-0064ad7bd060/go.mod h1:9B1MYPoLtP9tqjWxcbUNVpwxy68zOH/3EIP6c31dAM0=
+github.com/openshift/client-go v0.0.0-20241217083110-35abaf51555b h1:199fVpn6mYBw1x+dmCm9cITdlwRAj7vW7b95lD9CI0Q=
+github.com/openshift/client-go v0.0.0-20241217083110-35abaf51555b/go.mod h1:ZPJtGE5TTZko3gaqP5PBEjfiKrhBaDvMCTvtuOcOhr0=
+github.com/openshift/library-go v0.0.0-20241218091209-6018a90c28d0 h1:xf4uw8ShlOfRhtHziMv78jmEhWrWI+thDVJler1i3y4=
+github.com/openshift/library-go v0.0.0-20241218091209-6018a90c28d0/go.mod h1:eGSI6tp7yUVr4V2d0WrVt2l5s3iCwAh8Hi0RC9Fo16U=
 github.com/operator-framework/api v0.17.1 h1:J/6+Xj4IEV8C7hcirqUFwOiZAU3PbnJhWvB0/bB51c4=
 github.com/operator-framework/api v0.17.1/go.mod h1:kk8xJahHJR3bKqrA+A+1VIrhOTmyV76k+ARv+iV+u1Q=
 github.com/operator-framework/operator-lifecycle-manager v0.22.0 h1:7DEWOq24HQ0l5xPOXMhn17XaJACgwoipz+JfQ7QCXZw=

install/0000_00_cluster-version-operator_01_clusterversionoperators-DevPreviewNoUpgrade.crd.yaml

@@ -0,0 +1,90 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.openshift.io: https://github.com/openshift/api/pull/2044
+    api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-set: DevPreviewNoUpgrade
+  name: clusterversionoperators.operator.openshift.io
+spec:
+  group: operator.openshift.io
+  names:
+    kind: ClusterVersionOperator
+    listKind: ClusterVersionOperatorList
+    plural: clusterversionoperators
+    singular: clusterversionoperator
+  scope: Cluster
+  versions:
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: |-
+            ClusterVersionOperator holds cluster-wide information about the Cluster Version Operator.
+
+            Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: spec is the specification of the desired behavior of the
+                Cluster Version Operator.
+              properties:
+                operatorLogLevel:
+                  default: Normal
+                  description: |-
+                    operatorLogLevel is an intent based logging for the operator itself.  It does not give fine grained control, but it is a
+                    simple way to manage coarse grained logging choices that operators have to interpret for themselves.
+
+                    Valid values are: "Normal", "Debug", "Trace", "TraceAll".
+                    Defaults to "Normal".
+                  enum:
+                    - ""
+                    - Normal
+                    - Debug
+                    - Trace
+                    - TraceAll
+                  type: string
+              type: object
+            status:
+              description: status is the most recently observed status of the Cluster
+                Version Operator.
+              properties:
+                observedGeneration:
+                  description: |-
+                    observedGeneration represents the most recent generation observed by the operator and specifies the version of
+                    the spec field currently being synced.
+                  format: int64
+                  type: integer
+                  x-kubernetes-validations:
+                    - message: observedGeneration must only increase
+                      rule: self >= oldSelf
+              type: object
+          required:
+            - metadata
+            - spec
+          type: object
+          x-kubernetes-validations:
+            - message: ClusterVersionOperator is a singleton; the .metadata.name field
+                must be 'cluster'
+              rule: self.metadata.name == 'cluster'
+      served: true
+      storage: true
+      subresources:
+        status: {}

install/0000_00_cluster-version-operator_02_configuration-DevPreviewNoUpgrade.yaml

@@ -0,0 +1,11 @@
+apiVersion: operator.openshift.io/v1alpha1
+kind: ClusterVersionOperator
+metadata:
+  name: cluster
+  annotations:
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-set: DevPreviewNoUpgrade
+    release.openshift.io/create-only: "true"
+spec:
+  operatorLogLevel: Normal

pkg/cvo/cvo.go

@@ -31,6 +31,9 @@ import (
 	clientset "github.com/openshift/client-go/config/clientset/versioned"
 	configinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
 	configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
+	operatorclientset "github.com/openshift/client-go/operator/clientset/versioned"
+	operatorinformersv1alpha1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1"
+	operatorlistersv1alpha1 "github.com/openshift/client-go/operator/listers/operator/v1alpha1"
 	"github.com/openshift/library-go/pkg/manifest"
 	"github.com/openshift/library-go/pkg/verify"
 	"github.com/openshift/library-go/pkg/verify/store"
@@ -93,9 +96,10 @@ type Operator struct {
 	// releaseCreated, if set, is the timestamp of the current update.
 	releaseCreated time.Time
 
-	client        clientset.Interface
-	kubeClient    kubernetes.Interface
-	eventRecorder record.EventRecorder
+	client         clientset.Interface
+	operatorClient operatorclientset.Interface
+	kubeClient     kubernetes.Interface
+	eventRecorder  record.EventRecorder
 
 	// minimumUpdateCheckInterval is the minimum duration to check for updates from
 	// the update service.
@@ -112,19 +116,22 @@ type Operator struct {
 	// spec.
 	updateService string
 
-	cvLister              configlistersv1.ClusterVersionLister
-	coLister              configlistersv1.ClusterOperatorLister
-	cmConfigLister        listerscorev1.ConfigMapNamespaceLister
-	cmConfigManagedLister listerscorev1.ConfigMapNamespaceLister
-	proxyLister           configlistersv1.ProxyLister
-	cacheSynced           []cache.InformerSynced
+	cvLister               configlistersv1.ClusterVersionLister
+	coLister               configlistersv1.ClusterOperatorLister
+	cvoConfigurationLister operatorlistersv1alpha1.ClusterVersionOperatorLister
+	cmConfigLister         listerscorev1.ConfigMapNamespaceLister
+	cmConfigManagedLister  listerscorev1.ConfigMapNamespaceLister
+	proxyLister            configlistersv1.ProxyLister
+	cacheSynced            []cache.InformerSynced
 
 	// queue tracks applying updates to a cluster.
 	queue workqueue.TypedRateLimitingInterface[any]
 	// availableUpdatesQueue tracks checking for updates from the update server.
 	availableUpdatesQueue workqueue.TypedRateLimitingInterface[any]
 	// upgradeableQueue tracks checking for upgradeable.
 	upgradeableQueue workqueue.TypedRateLimitingInterface[any]
+	// configurationQueue tracks checking for configuration.
+	configurationQueue workqueue.TypedRateLimitingInterface[any]
 
 	// statusLock guards access to modifying available updates
 	statusLock       sync.Mutex
@@ -190,8 +197,10 @@ func New(
 	cmConfigInformer informerscorev1.ConfigMapInformer,
 	cmConfigManagedInformer informerscorev1.ConfigMapInformer,
 	proxyInformer configinformersv1.ProxyInformer,
+	cvoInformer operatorinformersv1alpha1.ClusterVersionOperatorInformer,
 	client clientset.Interface,
 	kubeClient kubernetes.Interface,
+	operatorClient operatorclientset.Interface,
 	exclude string,
 	clusterProfile string,
 	promqlTarget clusterconditions.PromQLTarget,
@@ -219,10 +228,12 @@ func New(
 
 		client:                client,
 		kubeClient:            kubeClient,
+		operatorClient:        operatorClient,
 		eventRecorder:         eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: namespace}),
 		queue:                 workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
 		availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
 		upgradeableQueue:      workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),
+		configurationQueue:    workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "configuration"}),
 
 		exclude:                   exclude,
 		clusterProfile:            clusterProfile,
@@ -248,13 +259,19 @@ func New(
 	if _, err := coInformer.Informer().AddEventHandler(optr.clusterOperatorEventHandler()); err != nil {
 		return nil, err
 	}
+	if _, err := cvoInformer.Informer().AddEventHandler(optr.clusterVersionOperatorEventHandler()); err != nil {
+		return nil, err
+	}
 
 	optr.coLister = coInformer.Lister()
 	optr.cacheSynced = append(optr.cacheSynced, coInformer.Informer().HasSynced)
 
 	optr.cvLister = cvInformer.Lister()
 	optr.cacheSynced = append(optr.cacheSynced, cvInformer.Informer().HasSynced)
 
+	optr.cvoConfigurationLister = cvoInformer.Lister()
+	optr.cacheSynced = append(optr.cacheSynced, cvoInformer.Informer().HasSynced)
+
 	optr.proxyLister = proxyInformer.Lister()
 	optr.cmConfigLister = cmConfigInformer.Lister().ConfigMaps(internal.ConfigNamespace)
 	optr.cmConfigManagedLister = cmConfigManagedInformer.Lister().ConfigMaps(internal.ConfigManagedNamespace)
@@ -446,6 +463,15 @@ func (optr *Operator) Run(runContext context.Context, shutdownContext context.Co
 		resultChannel <- asyncResult{name: "available updates"}
 	}()
 
+	resultChannelCount++
+	go func() {
+		defer utilruntime.HandleCrash()
+		wait.UntilWithContext(runContext, func(runContext context.Context) {
+			optr.worker(runContext, optr.configurationQueue, optr.configurationSync)
+		}, time.Second)
+		resultChannel <- asyncResult{name: "cvo configuration"}
+	}()
+
 	resultChannelCount++
 	go func() {
 		defer utilruntime.HandleCrash()
@@ -546,6 +572,26 @@ func (optr *Operator) clusterVersionEventHandler() cache.ResourceEventHandler {
 	}
 }
 
+// clusterOperatorEventHandler queues an update for the cluster version on any change to the given object.
+// Callers should use this with an informer.
+func (optr *Operator) clusterVersionOperatorEventHandler() cache.ResourceEventHandler {
+	workQueueKey := optr.queueKey()
+	return cache.ResourceEventHandlerFuncs{
+		AddFunc: func(_ interface{}) {
+			optr.configurationQueue.Add(workQueueKey)
+			klog.V(internal.Debug).Infof("ClusterVersionOperator resource was added; queuing a configuration sync")
+		},
+		UpdateFunc: func(_, _ interface{}) {
+			optr.configurationQueue.Add(workQueueKey)
+			klog.V(internal.Debug).Infof("ClusterVersionOperator resource was modified; queuing a configuration sync")
+		},
+		DeleteFunc: func(_ interface{}) {
+			optr.configurationQueue.Add(workQueueKey)
+			klog.V(internal.Debug).Infof("ClusterVersionOperator resource was deleted; queuing a configuration sync")
+		},
+	}
+}
+
 // clusterOperatorEventHandler queues an update for the cluster version on any change to the given object.
 // Callers should use this with an informer.
 func (optr *Operator) clusterOperatorEventHandler() cache.ResourceEventHandler {
@@ -774,6 +820,23 @@ func (optr *Operator) upgradeableSyncFunc(ignoreThrottlePeriod bool) func(_ cont
 	}
 }
 
+func (optr *Operator) configurationSync(ctx context.Context, _ string) error {
+	startTime := time.Now()
+	klog.V(2).Infof("Started syncing CVO configuration")
+	defer func() {
+		klog.V(2).Infof("Finished syncing CVO configuration (%v)", time.Since(startTime))
+	}()
+
+	config, err := optr.cvoConfigurationLister.Get(ClusterVersionOperatorConfigurationName)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	return optr.syncConfiguration(ctx, config)
+}
+
 // isOlderThanLastUpdate returns true if the cluster version is older than
 // the last update we saw.
 func (optr *Operator) isOlderThanLastUpdate(config *configv1.ClusterVersion) bool {

pkg/cvo/cvoconfiguration.go

@@ -0,0 +1,59 @@
+package cvo
+
+import (
+	"context"
+	"fmt"
+
+	operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1"
+	"github.com/openshift/library-go/pkg/operator/loglevel"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+
+	"github.com/openshift/cluster-version-operator/pkg/internal"
+)
+
+const ClusterVersionOperatorConfigurationName = "cluster"
+
+func (optr *Operator) syncConfiguration(ctx context.Context, config *operatorv1alpha1.ClusterVersionOperator) error {
+	if !optr.enabledFeatureGates.CVOConfiguration() {
+		// TODO: Change to `internal.Debug` after testing
+		// TODO: Reconsider placement of FeatureGate checks
+		klog.V(internal.Normal).Infof("The ClusterVersionOperatorConfiguration feature gate is disabled; skipping sync configuration")
+		return nil
+	}
+
+	if config.Status.ObservedGeneration != config.Generation {
+		config.Status.ObservedGeneration = config.Generation
+		_, err := optr.operatorClient.OperatorV1alpha1().ClusterVersionOperators().UpdateStatus(ctx, config, metav1.UpdateOptions{})
+		if err != nil {
+			err = fmt.Errorf("Failed to update the ClusterVersionOperator resource, err=%w", err)
+			klog.Error(err)
+			return err
+		}
+	}
+
+	current, notFound := loglevel.GetLogLevel()
+	if notFound {
+		klog.Warningf("The current log level could not be found; an attempt to set the log level to desired level will be made")
+	}
+
+	desired := config.Spec.OperatorLogLevel
+	if !notFound && current == desired {
+		klog.Infof("No need to update the current CVO log level '%s'; it is already set to the desired value", current)
+	} else {
+		err := loglevel.SetLogLevel(desired)
+		if err != nil {
+			err = fmt.Errorf("Failed to set the log level to %s, err=%w", desired, err)
+			klog.Error(err)
+			return err
+		}
+		klog.Infof("Successfully updated the log level from '%s' to '%s'", current, desired)
+
+		// TODO: Remove when done
+		klog.V(internal.Normal).Infof("This line is only printed at the 'Normal' log level and above")
+		klog.V(internal.Debug).Infof("This line is only printed at the 'Debug' log level and above")
+		klog.V(internal.Trace).Infof("This line is only printed at the 'Trace' log level and above")
+		klog.V(internal.TraceAll).Infof("This line is only printed at the 'TraceAll' log level and above")
+	}
+	return nil
+}