Skip to content

Commit

Permalink
Remove dynamic log level changing
Browse files Browse the repository at this point in the history
  • Loading branch information
jonathan-innis committed Sep 27, 2023
1 parent 7bda4a7 commit 725cf8c
Show file tree
Hide file tree
Showing 7 changed files with 269 additions and 153 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21
require (
github.com/Pallinder/go-randomdata v1.2.0
github.com/avast/retry-go v3.0.0+incompatible
github.com/blendle/zapdriver v1.3.1
github.com/deckarep/golang-set v1.8.0
github.com/go-logr/logr v1.2.4
github.com/go-logr/zapr v1.2.4
Expand All @@ -18,6 +19,7 @@ require (
github.com/samber/lo v1.38.1
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.26.0
golang.org/x/sync v0.3.0
golang.org/x/text v0.13.0
golang.org/x/time v0.3.0
k8s.io/api v0.26.6
Expand All @@ -37,7 +39,6 @@ require (
contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/blendle/zapdriver v1.3.1 // indirect
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down Expand Up @@ -81,7 +82,6 @@ require (
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.11.0 // indirect
golang.org/x/tools v0.12.0 // indirect
Expand Down
25 changes: 6 additions & 19 deletions pkg/operator/injection/injection.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,11 @@ import (

"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"knative.dev/pkg/system"

Expand All @@ -46,20 +47,6 @@ func GetOptions(ctx context.Context) options.Options {
return retval.(options.Options)
}

type configKey struct{}

func WithConfig(ctx context.Context, config *rest.Config) context.Context {
return context.WithValue(ctx, configKey{}, config)
}

func GetConfig(ctx context.Context) *rest.Config {
retval := ctx.Value(configKey{})
if retval == nil {
return nil
}
return retval.(*rest.Config)
}

type controllerNameKeyType struct{}

var controllerNameKey = controllerNameKeyType{}
Expand Down Expand Up @@ -89,22 +76,22 @@ func WithSettingsOrDie(ctx context.Context, kubernetesInterface kubernetes.Inter
factory.Start(cancelCtx.Done())

for _, setting := range settings {
cm := lo.Must(waitForConfigMap(ctx, setting.ConfigMap(), informer))
cm := lo.Must(WaitForConfigMap(ctx, setting.ConfigMap(), informer))
ctx = lo.Must(setting.Inject(ctx, cm))
}
return ctx
}

// waitForConfigMap waits until all registered configMaps in the settingsStore are created
func waitForConfigMap(ctx context.Context, name string, informer cache.SharedIndexInformer) (*v1.ConfigMap, error) {
// WaitForConfigMap waits until all registered configMaps in the settingsStore are created
func WaitForConfigMap(ctx context.Context, name string, informer cache.SharedIndexInformer) (*v1.ConfigMap, error) {
for {
configMap, exists, err := informer.GetStore().GetByKey(types.NamespacedName{Namespace: system.Namespace(), Name: name}.String())
if configMap != nil && exists && err == nil {
return configMap.(*v1.ConfigMap), nil
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("context canceled")
return nil, fmt.Errorf("context canceled, %w", errors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, types.NamespacedName{Namespace: system.Namespace(), Name: name}.String()))
case <-time.After(time.Millisecond * 500):
}
}
Expand Down
84 changes: 0 additions & 84 deletions pkg/operator/logger.go

This file was deleted.

153 changes: 153 additions & 0 deletions pkg/operator/logging/logging.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
"context"
"encoding/json"
"fmt"
"log"
"time"

"github.com/blendle/zapdriver"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"github.com/samber/lo"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zapio"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
"knative.dev/pkg/changeset"
"knative.dev/pkg/logging"
"knative.dev/pkg/logging/logkey"
"knative.dev/pkg/system"

"github.com/aws/karpenter-core/pkg/operator/injection"
)

const (
loggerCfgConfigMapName = "config-logging"
loggerCfgConfigMapKey = "zap-logger-config"
)

func DefaultZapConfig() zap.Config {
cfg := zapdriver.NewProductionConfig()
cfg.EncoderConfig.EncodeDuration = zapcore.StringDurationEncoder
return cfg
}

// NewLogger returns a configured *zap.SugaredLogger
func NewLogger(ctx context.Context, component string, kubernetesInterface kubernetes.Interface) *zap.SugaredLogger {
if logger := loggerFromConfigMap(ctx, component, kubernetesInterface); logger != nil {
if injection.GetOptions(ctx).LogLevel != nil {
log.Fatalf(`--log-level cannot be set while using the "config-logging" ConfigMap`)
}
return logger
}
return defaultLogger(ctx)
}

func withCommit(logger *zap.SugaredLogger) *zap.SugaredLogger {
revision := changeset.Get()
if revision == changeset.Unknown {
logger.Info("Unable to read vcs.revision from binary")
return logger
}
// Enrich logs with the components git revision.
return logger.With(zap.String(logkey.Commit, revision))
}

func defaultLogger(ctx context.Context) *zap.SugaredLogger {
cfg := DefaultZapConfig()
if injection.GetOptions(ctx).LogLevel != nil {
cfg.Level = lo.FromPtr(injection.GetOptions(ctx).LogLevel)
}
return withCommit(lo.Must(cfg.Build()).Sugar())
}

func loggerFromConfigMap(ctx context.Context, component string, kubernetesInterface kubernetes.Interface) *zap.SugaredLogger {
cancelCtx, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()

factory := informers.NewSharedInformerFactoryWithOptions(kubernetesInterface, time.Second*30, informers.WithNamespace(system.Namespace()))
informer := factory.Core().V1().ConfigMaps().Informer()
factory.Start(cancelCtx.Done())

cm, err := injection.WaitForConfigMap(ctx, loggerCfgConfigMapName, informer)
if err != nil {
if errors.IsNotFound(err) {
return nil
}
log.Fatalf("retrieving logging config map from %q", types.NamespacedName{Namespace: system.Namespace(), Name: loggerCfgConfigMapName})
}
cfg := DefaultZapConfig()
lo.Must0(json.Unmarshal([]byte(cm.Data[loggerCfgConfigMapKey]), &cfg))

if v := cm.Data[fmt.Sprintf("loglevel.%s", component)]; v != "" {
cfg.Level = lo.Must(zap.ParseAtomicLevel(v))
}
if injection.GetOptions(ctx).LogLevel != nil {
cfg.Level = lo.FromPtr(injection.GetOptions(ctx).LogLevel)
}
return withCommit(lo.Must(cfg.Build()).Sugar())
}

// ConfigureGlobalLoggers sets up any package-wide loggers like "log" or "klog" that are utilized by other packages
// to use the configured *zap.SugaredLogger from the context
func ConfigureGlobalLoggers(ctx context.Context) {
klog.SetLogger(zapr.NewLogger(logging.FromContext(ctx).Desugar()))
w := &zapio.Writer{Log: logging.FromContext(ctx).Desugar(), Level: zap.DebugLevel}
log.SetFlags(0)
log.SetOutput(w)
rest.SetDefaultWarningHandler(&logging.WarningHandler{Logger: logging.FromContext(ctx)})
}

type ignoreDebugEventsSink struct {
name string
sink logr.LogSink
}

func (i ignoreDebugEventsSink) Init(ri logr.RuntimeInfo) {
i.sink.Init(ri)
}
func (i ignoreDebugEventsSink) Enabled(level int) bool { return i.sink.Enabled(level) }
func (i ignoreDebugEventsSink) Info(level int, msg string, keysAndValues ...interface{}) {
// ignore debug "events" logs
if level == 1 && i.name == "events" {
return
}
i.sink.Info(level, msg, keysAndValues...)
}
func (i ignoreDebugEventsSink) Error(err error, msg string, keysAndValues ...interface{}) {
i.sink.Error(err, msg, keysAndValues...)
}
func (i ignoreDebugEventsSink) WithValues(keysAndValues ...interface{}) logr.LogSink {
return i.sink.WithValues(keysAndValues...)
}
func (i ignoreDebugEventsSink) WithName(name string) logr.LogSink {
return &ignoreDebugEventsSink{name: name, sink: i.sink.WithName(name)}
}

// IgnoreDebugEvents wraps the logger with one that ignores any debug logs coming from a logger named "events". This
// prevents every event we write from creating a debug log which spams the log file during scale-ups due to recording
// pod scheduling decisions as events for visibility.
func IgnoreDebugEvents(logger logr.Logger) logr.Logger {
return logr.New(&ignoreDebugEventsSink{sink: logger.GetSink()})
}
Loading

0 comments on commit 725cf8c

Please sign in to comment.