From 725cf8cf261bb4ab872ff9f4bc1cf62efeed0721 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 26 Sep 2023 23:43:34 -0700 Subject: [PATCH] Remove dynamic log level changing --- go.mod | 4 +- pkg/operator/injection/injection.go | 25 ++--- pkg/operator/logger.go | 84 --------------- pkg/operator/logging/logging.go | 153 ++++++++++++++++++++++++++++ pkg/operator/operator.go | 58 +++-------- pkg/operator/options/options.go | 3 + pkg/webhooks/webhooks.go | 95 ++++++++++++++++- 7 files changed, 269 insertions(+), 153 deletions(-) delete mode 100644 pkg/operator/logger.go create mode 100644 pkg/operator/logging/logging.go diff --git a/go.mod b/go.mod index 5137f8a524..08c4e50256 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/avast/retry-go v3.0.0+incompatible + github.com/blendle/zapdriver v1.3.1 github.com/deckarep/golang-set v1.8.0 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 @@ -18,6 +19,7 @@ require ( github.com/samber/lo v1.38.1 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 + golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 golang.org/x/time v0.3.0 k8s.io/api v0.26.6 @@ -37,7 +39,6 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/blendle/zapdriver v1.3.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -81,7 +82,6 @@ require ( golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.11.0 // indirect golang.org/x/tools v0.12.0 // indirect diff --git a/pkg/operator/injection/injection.go b/pkg/operator/injection/injection.go index c399eaafa9..84db5d3a0a 100644 --- a/pkg/operator/injection/injection.go +++ b/pkg/operator/injection/injection.go @@ -21,10 +21,11 @@ import ( "github.com/samber/lo" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "knative.dev/pkg/system" @@ -46,20 +47,6 @@ func GetOptions(ctx context.Context) options.Options { return retval.(options.Options) } -type configKey struct{} - -func WithConfig(ctx context.Context, config *rest.Config) context.Context { - return context.WithValue(ctx, configKey{}, config) -} - -func GetConfig(ctx context.Context) *rest.Config { - retval := ctx.Value(configKey{}) - if retval == nil { - return nil - } - return retval.(*rest.Config) -} - type controllerNameKeyType struct{} var controllerNameKey = controllerNameKeyType{} @@ -89,14 +76,14 @@ func WithSettingsOrDie(ctx context.Context, kubernetesInterface kubernetes.Inter factory.Start(cancelCtx.Done()) for _, setting := range settings { - cm := lo.Must(waitForConfigMap(ctx, setting.ConfigMap(), informer)) + cm := lo.Must(WaitForConfigMap(ctx, setting.ConfigMap(), informer)) ctx = lo.Must(setting.Inject(ctx, cm)) } return ctx } -// waitForConfigMap waits until all registered configMaps in the settingsStore are created -func waitForConfigMap(ctx context.Context, name string, informer cache.SharedIndexInformer) (*v1.ConfigMap, error) { +// WaitForConfigMap waits until all registered configMaps in the settingsStore are created +func WaitForConfigMap(ctx context.Context, name string, informer cache.SharedIndexInformer) (*v1.ConfigMap, error) { for { configMap, exists, err := informer.GetStore().GetByKey(types.NamespacedName{Namespace: system.Namespace(), Name: name}.String()) if configMap != nil && exists && err == nil { @@ -104,7 +91,7 @@ func waitForConfigMap(ctx context.Context, name string, informer cache.SharedInd } select { case <-ctx.Done(): - return nil, fmt.Errorf("context canceled") + return nil, fmt.Errorf("context canceled, %w", errors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, types.NamespacedName{Namespace: system.Namespace(), Name: name}.String())) case <-time.After(time.Millisecond * 500): } } diff --git a/pkg/operator/logger.go b/pkg/operator/logger.go deleted file mode 100644 index a1911e915e..0000000000 --- a/pkg/operator/logger.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operator - -import ( - "context" - "log" - - "github.com/go-logr/logr" - "github.com/go-logr/zapr" - "go.uber.org/zap" - "go.uber.org/zap/zapio" - "k8s.io/client-go/rest" - "k8s.io/klog/v2" - "knative.dev/pkg/configmap/informer" - "knative.dev/pkg/injection" - "knative.dev/pkg/injection/sharedmain" - "knative.dev/pkg/logging" -) - -// NewLogger returns a configured *zap.SugaredLogger. The logger is -// configured by the ConfigMap `config-logging` and live updates the level. -func NewLogger(ctx context.Context, componentName string, config *rest.Config, cmw *informer.InformedWatcher) *zap.SugaredLogger { - ctx, startInformers := injection.EnableInjectionOrDie(logging.WithLogger(ctx, zap.NewNop().Sugar()), config) - logger, atomicLevel := sharedmain.SetupLoggerOrDie(ctx, componentName) - rest.SetDefaultWarningHandler(&logging.WarningHandler{Logger: logger}) - sharedmain.WatchLoggingConfigOrDie(ctx, cmw, logger, atomicLevel, componentName) - startInformers() - return logger -} - -// ConfigureGlobalLoggers sets up any package-wide loggers like "log" or "klog" that are utilized by other packages -// to use the configured *zap.SugaredLogger from the context -func ConfigureGlobalLoggers(ctx context.Context) { - klog.SetLogger(zapr.NewLogger(logging.FromContext(ctx).Desugar())) - w := &zapio.Writer{Log: logging.FromContext(ctx).Desugar(), Level: zap.DebugLevel} - log.SetFlags(0) - log.SetOutput(w) -} - -type ignoreDebugEventsSink struct { - name string - sink logr.LogSink -} - -func (i ignoreDebugEventsSink) Init(ri logr.RuntimeInfo) { - i.sink.Init(ri) -} -func (i ignoreDebugEventsSink) Enabled(level int) bool { return i.sink.Enabled(level) } -func (i ignoreDebugEventsSink) Info(level int, msg string, keysAndValues ...interface{}) { - // ignore debug "events" logs - if level == 1 && i.name == "events" { - return - } - i.sink.Info(level, msg, keysAndValues...) -} -func (i ignoreDebugEventsSink) Error(err error, msg string, keysAndValues ...interface{}) { - i.sink.Error(err, msg, keysAndValues...) -} -func (i ignoreDebugEventsSink) WithValues(keysAndValues ...interface{}) logr.LogSink { - return i.sink.WithValues(keysAndValues...) -} -func (i ignoreDebugEventsSink) WithName(name string) logr.LogSink { - return &ignoreDebugEventsSink{name: name, sink: i.sink.WithName(name)} -} - -// ignoreDebugEvents wraps the logger with one that ignores any debug logs coming from a logger named "events". This -// prevents every event we write from creating a debug log which spams the log file during scale-ups due to recording -// pod scheduling decisions as events for visibility. -func ignoreDebugEvents(logger logr.Logger) logr.Logger { - return logr.New(&ignoreDebugEventsSink{sink: logger.GetSink()}) -} diff --git a/pkg/operator/logging/logging.go b/pkg/operator/logging/logging.go new file mode 100644 index 0000000000..d120b35e20 --- /dev/null +++ b/pkg/operator/logging/logging.go @@ -0,0 +1,153 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logging + +import ( + "context" + "encoding/json" + "fmt" + "log" + "time" + + "github.com/blendle/zapdriver" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" + "github.com/samber/lo" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zapio" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "knative.dev/pkg/changeset" + "knative.dev/pkg/logging" + "knative.dev/pkg/logging/logkey" + "knative.dev/pkg/system" + + "github.com/aws/karpenter-core/pkg/operator/injection" +) + +const ( + loggerCfgConfigMapName = "config-logging" + loggerCfgConfigMapKey = "zap-logger-config" +) + +func DefaultZapConfig() zap.Config { + cfg := zapdriver.NewProductionConfig() + cfg.EncoderConfig.EncodeDuration = zapcore.StringDurationEncoder + return cfg +} + +// NewLogger returns a configured *zap.SugaredLogger +func NewLogger(ctx context.Context, component string, kubernetesInterface kubernetes.Interface) *zap.SugaredLogger { + if logger := loggerFromConfigMap(ctx, component, kubernetesInterface); logger != nil { + if injection.GetOptions(ctx).LogLevel != nil { + log.Fatalf(`--log-level cannot be set while using the "config-logging" ConfigMap`) + } + return logger + } + return defaultLogger(ctx) +} + +func withCommit(logger *zap.SugaredLogger) *zap.SugaredLogger { + revision := changeset.Get() + if revision == changeset.Unknown { + logger.Info("Unable to read vcs.revision from binary") + return logger + } + // Enrich logs with the components git revision. + return logger.With(zap.String(logkey.Commit, revision)) +} + +func defaultLogger(ctx context.Context) *zap.SugaredLogger { + cfg := DefaultZapConfig() + if injection.GetOptions(ctx).LogLevel != nil { + cfg.Level = lo.FromPtr(injection.GetOptions(ctx).LogLevel) + } + return withCommit(lo.Must(cfg.Build()).Sugar()) +} + +func loggerFromConfigMap(ctx context.Context, component string, kubernetesInterface kubernetes.Interface) *zap.SugaredLogger { + cancelCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + factory := informers.NewSharedInformerFactoryWithOptions(kubernetesInterface, time.Second*30, informers.WithNamespace(system.Namespace())) + informer := factory.Core().V1().ConfigMaps().Informer() + factory.Start(cancelCtx.Done()) + + cm, err := injection.WaitForConfigMap(ctx, loggerCfgConfigMapName, informer) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + log.Fatalf("retrieving logging config map from %q", types.NamespacedName{Namespace: system.Namespace(), Name: loggerCfgConfigMapName}) + } + cfg := DefaultZapConfig() + lo.Must0(json.Unmarshal([]byte(cm.Data[loggerCfgConfigMapKey]), &cfg)) + + if v := cm.Data[fmt.Sprintf("loglevel.%s", component)]; v != "" { + cfg.Level = lo.Must(zap.ParseAtomicLevel(v)) + } + if injection.GetOptions(ctx).LogLevel != nil { + cfg.Level = lo.FromPtr(injection.GetOptions(ctx).LogLevel) + } + return withCommit(lo.Must(cfg.Build()).Sugar()) +} + +// ConfigureGlobalLoggers sets up any package-wide loggers like "log" or "klog" that are utilized by other packages +// to use the configured *zap.SugaredLogger from the context +func ConfigureGlobalLoggers(ctx context.Context) { + klog.SetLogger(zapr.NewLogger(logging.FromContext(ctx).Desugar())) + w := &zapio.Writer{Log: logging.FromContext(ctx).Desugar(), Level: zap.DebugLevel} + log.SetFlags(0) + log.SetOutput(w) + rest.SetDefaultWarningHandler(&logging.WarningHandler{Logger: logging.FromContext(ctx)}) +} + +type ignoreDebugEventsSink struct { + name string + sink logr.LogSink +} + +func (i ignoreDebugEventsSink) Init(ri logr.RuntimeInfo) { + i.sink.Init(ri) +} +func (i ignoreDebugEventsSink) Enabled(level int) bool { return i.sink.Enabled(level) } +func (i ignoreDebugEventsSink) Info(level int, msg string, keysAndValues ...interface{}) { + // ignore debug "events" logs + if level == 1 && i.name == "events" { + return + } + i.sink.Info(level, msg, keysAndValues...) +} +func (i ignoreDebugEventsSink) Error(err error, msg string, keysAndValues ...interface{}) { + i.sink.Error(err, msg, keysAndValues...) +} +func (i ignoreDebugEventsSink) WithValues(keysAndValues ...interface{}) logr.LogSink { + return i.sink.WithValues(keysAndValues...) +} +func (i ignoreDebugEventsSink) WithName(name string) logr.LogSink { + return &ignoreDebugEventsSink{name: name, sink: i.sink.WithName(name)} +} + +// IgnoreDebugEvents wraps the logger with one that ignores any debug logs coming from a logger named "events". This +// prevents every event we write from creating a debug log which spams the log file during scale-ups due to recording +// pod scheduling decisions as events for visibility. +func IgnoreDebugEvents(logger logr.Logger) logr.Logger { + return logr.New(&ignoreDebugEventsSink{sink: logger.GetSink()}) +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 6566461c97..039f5c41eb 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -17,7 +17,6 @@ package operator import ( "context" "fmt" - "io" "net/http" "sync" "time" @@ -31,10 +30,8 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/util/flowcontrol" "k8s.io/utils/clock" - "knative.dev/pkg/configmap/informer" knativeinjection "knative.dev/pkg/injection" - "knative.dev/pkg/injection/sharedmain" - "knative.dev/pkg/logging" + knativelogging "knative.dev/pkg/logging" "knative.dev/pkg/signals" "knative.dev/pkg/system" "knative.dev/pkg/webhook" @@ -49,8 +46,10 @@ import ( "github.com/aws/karpenter-core/pkg/events" corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/injection" + "github.com/aws/karpenter-core/pkg/operator/logging" "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" + "github.com/aws/karpenter-core/pkg/webhooks" ) const ( @@ -73,9 +72,6 @@ func NewOperator() (context.Context, *Operator) { // Root Context ctx := signals.NewContext() ctx = knativeinjection.WithNamespaceScope(ctx, system.Namespace()) - // TODO: This can be removed if we eventually decide that we need leader election. Having leader election has resulted in the webhook - // having issues described in https://github.com/aws/karpenter/issues/2562 so these issues need to be resolved if this line is removed - ctx = sharedmain.WithHADisabled(ctx) // Disable leader election for webhook // Options opts := options.New().MustParse() @@ -96,20 +92,18 @@ func NewOperator() (context.Context, *Operator) { // Client kubernetesInterface := kubernetes.NewForConfigOrDie(config) - configMapWatcher := informer.NewInformedWatcher(kubernetesInterface, system.Namespace()) - lo.Must0(configMapWatcher.Start(ctx.Done())) // Logging - logger := NewLogger(ctx, component, config, configMapWatcher) - ctx = logging.WithLogger(ctx, logger) - ConfigureGlobalLoggers(ctx) + logger := logging.NewLogger(ctx, component, kubernetesInterface) + ctx = knativelogging.WithLogger(ctx, logger) + logging.ConfigureGlobalLoggers(ctx) // Inject settings from the ConfigMap(s) into the context ctx = injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...) // Manager mgr, err := controllerruntime.NewManager(config, controllerruntime.Options{ - Logger: ignoreDebugEvents(zapr.NewLogger(logger.Desugar())), + Logger: logging.IgnoreDebugEvents(zapr.NewLogger(logger.Desugar())), LeaderElection: opts.EnableLeaderElection, LeaderElectionID: "karpenter-leader-election", LeaderElectionResourceLock: resourcelock.LeasesResourceLock, @@ -119,9 +113,8 @@ func NewOperator() (context.Context, *Operator) { HealthProbeBindAddress: fmt.Sprintf(":%d", opts.HealthProbePort), BaseContext: func() context.Context { ctx := context.Background() - ctx = logging.WithLogger(ctx, logger) + ctx = knativelogging.WithLogger(ctx, logger) ctx = injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...) - ctx = injection.WithConfig(ctx, config) ctx = injection.WithOptions(ctx, *opts) return ctx }, @@ -169,11 +162,11 @@ func (o *Operator) WithControllers(ctx context.Context, controllers ...corecontr return o } -func (o *Operator) WithWebhooks(ctx context.Context, webhooks ...knativeinjection.ControllerConstructor) *Operator { +func (o *Operator) WithWebhooks(ctx context.Context, ctors ...knativeinjection.ControllerConstructor) *Operator { if !injection.GetOptions(ctx).DisableWebhook { - o.webhooks = append(o.webhooks, webhooks...) - lo.Must0(o.Manager.AddReadyzCheck("webhooks", webhookChecker(ctx))) - lo.Must0(o.Manager.AddHealthzCheck("webhooks", webhookChecker(ctx))) + o.webhooks = append(o.webhooks, ctors...) + lo.Must0(o.Manager.AddReadyzCheck("webhooks", webhooks.HealthProbe(ctx))) + lo.Must0(o.Manager.AddHealthzCheck("webhooks", webhooks.HealthProbe(ctx))) } return o } @@ -186,36 +179,13 @@ func (o *Operator) Start(ctx context.Context) { lo.Must0(o.Manager.Start(ctx)) }() if injection.GetOptions(ctx).DisableWebhook { - logging.FromContext(ctx).Infof("webhook disabled") + knativelogging.FromContext(ctx).Infof("webhook disabled") } else { wg.Add(1) go func() { defer wg.Done() - sharedmain.MainWithConfig(sharedmain.WithHealthProbesDisabled(ctx), "webhook", o.GetConfig(), o.webhooks...) + webhooks.Start(ctx, o.GetConfig(), o.KubernetesInterface, o.webhooks...) }() } wg.Wait() } - -func webhookChecker(ctx context.Context) healthz.Checker { - // TODO: Add knative health check port for webhooks when health port can be configured - // Issue: https://github.com/knative/pkg/issues/2765 - return func(req *http.Request) (err error) { - res, err := http.Get(fmt.Sprintf("http://localhost:%d", injection.GetOptions(ctx).WebhookPort)) - // If the webhook connection errors out, liveness/readiness should fail - if err != nil { - return err - } - // Close the body to avoid leaking file descriptors - // Always read the body so we can re-use the connection: https://stackoverflow.com/questions/17948827/reusing-http-connections-in-go - _, _ = io.ReadAll(res.Body) - res.Body.Close() - - // If there is a server-side error or path not found, - // consider liveness to have failed - if res.StatusCode >= 500 || res.StatusCode == 404 { - return fmt.Errorf("webhook probe failed with status code %d", res.StatusCode) - } - return nil - } -} diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 2266fb7f30..a52e8399fe 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -20,6 +20,8 @@ import ( "os" "runtime/debug" + "go.uber.org/zap" + "github.com/aws/karpenter-core/pkg/utils/env" ) @@ -36,6 +38,7 @@ type Options struct { KubeClientBurst int EnableProfiling bool EnableLeaderElection bool + LogLevel *zap.AtomicLevel MemoryLimit int64 } diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index f4d0e0dbd0..212cf016a6 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -16,20 +16,39 @@ package webhooks import ( "context" + "errors" + "fmt" + "io" + "net/http" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" knativeinjection "knative.dev/pkg/injection" - "knative.dev/pkg/logging" + "knative.dev/pkg/injection/sharedmain" + knativelogging "knative.dev/pkg/logging" + "knative.dev/pkg/webhook" "knative.dev/pkg/webhook/certificates" "knative.dev/pkg/webhook/configmaps" "knative.dev/pkg/webhook/resourcesemantics" "knative.dev/pkg/webhook/resourcesemantics/validation" + "sigs.k8s.io/controller-runtime/pkg/healthz" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/operator/injection" + "github.com/aws/karpenter-core/pkg/operator/logging" ) +const component = "webhook" + +var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ + v1alpha5.SchemeGroupVersion.WithKind("Provisioner"): &v1alpha5.Provisioner{}, +} + func NewWebhooks() []knativeinjection.ControllerConstructor { return []knativeinjection.ControllerConstructor{ certificates.NewController, @@ -53,11 +72,79 @@ func NewConfigValidationWebhook(ctx context.Context, _ configmap.Watcher) *contr "validation.webhook.config.karpenter.sh", "/validate/config.karpenter.sh", configmap.Constructors{ - logging.ConfigMapName(): logging.NewConfigFromConfigMap, + knativelogging.ConfigMapName(): knativelogging.NewConfigFromConfigMap, }, ) } -var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ - v1alpha5.SchemeGroupVersion.WithKind("Provisioner"): &v1alpha5.Provisioner{}, +func Start(ctx context.Context, cfg *rest.Config, kubernetesInterface kubernetes.Interface, ctors ...knativeinjection.ControllerConstructor) { + logger := logging.NewLogger(ctx, component, kubernetesInterface) + ctx = knativelogging.WithLogger(ctx, logger) + ctx, startInformers := knativeinjection.EnableInjectionOrDie(ctx, cfg) + cmw := sharedmain.SetupConfigMapWatchOrDie(ctx, knativelogging.FromContext(ctx)) + controllers, webhooks := sharedmain.ControllersAndWebhooksFromCtors(ctx, cmw, ctors...) + if err := cmw.Start(ctx.Done()); err != nil { + knativelogging.FromContext(ctx).Fatalw("Failed to start configuration manager", zap.Error(err)) + } + eg, egCtx := errgroup.WithContext(ctx) + + // If we have one or more admission controllers, then start the webhook + // and pass them in. + var wh *webhook.Webhook + var err error + if len(webhooks) > 0 { + // Register webhook metrics + webhook.RegisterMetrics() + + wh, err = webhook.New(ctx, webhooks) + if err != nil { + knativelogging.FromContext(ctx).Fatalw("Failed to create webhook", zap.Error(err)) + } + eg.Go(func() error { + return wh.Run(ctx.Done()) + }) + } + + // Start the injection clients and informers. + startInformers() + + // Wait for webhook informers to sync. + if wh != nil { + wh.InformersHaveSynced() + } + knativelogging.FromContext(ctx).Info("Starting controllers...") + eg.Go(func() error { + return controller.StartAll(ctx, controllers...) + }) + // This will block until either a signal arrives or one of the grouped functions + // returns an error. + <-egCtx.Done() + + // Don't forward ErrServerClosed as that indicates we're already shutting down. + if err := eg.Wait(); err != nil && !errors.Is(err, http.ErrServerClosed) { + knativelogging.FromContext(ctx).Errorw("Error while running server", zap.Error(err)) + } +} + +func HealthProbe(ctx context.Context) healthz.Checker { + // TODO: Add knative health check port for webhooks when health port can be configured + // Issue: https://github.com/knative/pkg/issues/2765 + return func(req *http.Request) (err error) { + res, err := http.Get(fmt.Sprintf("http://localhost:%d", injection.GetOptions(ctx).WebhookPort)) + // If the webhook connection errors out, liveness/readiness should fail + if err != nil { + return err + } + // Close the body to avoid leaking file descriptors + // Always read the body so we can re-use the connection: https://stackoverflow.com/questions/17948827/reusing-http-connections-in-go + _, _ = io.ReadAll(res.Body) + res.Body.Close() + + // If there is a server-side error or path not found, + // consider liveness to have failed + if res.StatusCode >= 500 || res.StatusCode == 404 { + return fmt.Errorf("webhook probe failed with status code %d", res.StatusCode) + } + return nil + } }