Upgrade dskit to the version without log.Interface and logrus

Signed-off-by: Yuri Nikolic <[email protected]>

CHANGELOG.md

@@ -1,4 +1,5 @@
 ## main / unreleased
+* [CHANGE] `RateLimitedLogger` has been replaced with `dskit`'s equivalent. The former has been removed from `tempo`. #2796
 * [FEATURE] Add the `/api/status/buildinfo` endpoint [#2702](https://github.com/grafana/tempo/pull/2702) (@fabrizio-grafana)
 * [FEATURE] New encoding vParquet3 with support for dedicated attribute columns (@mapno, @stoewer) [#2649](https://github.com/grafana/tempo/pull/2649)
 * [FEATURE] Add filtering support to Generic Forwarding [#2742](https://github.com/grafana/tempo/pull/2742) (@Blinkuu)

cmd/tempo/main.go

@@ -71,7 +71,7 @@ func main() {
 		level.Error(log.Logger).Log("msg", "invalid log level")
 		os.Exit(1)
 	}
-	log.InitLogger(&config.Server)
+	config.Server.Log = log.InitLogger(config.Server.LogFormat, config.Server.LogLevel)
 
 	// Verifying the config's validity and log warnings now that the logger is initialized
 	isValid := configIsValid(config)

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/alecthomas/kong v0.8.0
 	github.com/alicebob/miniredis/v2 v2.21.0
-	github.com/aws/aws-sdk-go v1.44.220
+	github.com/aws/aws-sdk-go v1.44.321
 	github.com/cespare/xxhash v1.1.0
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/cristalhq/hedgedhttp v0.7.0
@@ -30,7 +30,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
-	github.com/grafana/dskit v0.0.0-20230808152345-e772133474bd
+	github.com/grafana/dskit v0.0.0-20230817093031-e6b085110957
 	github.com/grafana/e2e v0.1.1-0.20221018202458-cffd2bb71c7b
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/hashicorp/go-hclog v1.4.0
@@ -60,7 +60,7 @@ require (
 	github.com/prometheus/prometheus v1.8.2-0.20221021121301-51a44e6657c3
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
-	github.com/sirupsen/logrus v1.9.0
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/sony/gobreaker v0.4.1
 	github.com/spf13/viper v1.14.0
 	github.com/stretchr/testify v1.8.4
@@ -88,7 +88,7 @@ require (
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/time v0.1.0
+	golang.org/x/time v0.1.0 // indirect
 	google.golang.org/api v0.125.0
 	google.golang.org/grpc v1.56.2
 	google.golang.org/protobuf v1.31.0
@@ -264,7 +264,6 @@ require (
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/weaveworks/promrus v1.2.0 // indirect
 	github.com/willf/bitset v1.1.11 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect

go.sum

@@ -476,8 +476,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo=
-github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.321 h1:iXwFLxWjZPjYqjPq0EcCs46xX7oDLEELte1+BzgpKk8=
+github.com/aws/aws-sdk-go v1.44.321/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
 github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo=
 github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw=
@@ -826,8 +826,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/grafana/dskit v0.0.0-20230808152345-e772133474bd h1:nsjY1yHyUVAkDLDwhd+3f91ox45Wv+xRxrEYxDeF6Y8=
-github.com/grafana/dskit v0.0.0-20230808152345-e772133474bd/go.mod h1:hbcgTCAMxGfZmjbieS4VZpnJjzvcWlI8V7/uTJ9nTMM=
+github.com/grafana/dskit v0.0.0-20230817093031-e6b085110957 h1:qpbwClUfInqZk+yWG2ESnfjgPU7E10GqH8eh4uJI1+8=
+github.com/grafana/dskit v0.0.0-20230817093031-e6b085110957/go.mod h1:3u7fr4hmOhuUL9Yc1QP/oa3za73kxvqJnRJH4BA5fOM=
 github.com/grafana/e2e v0.1.1-0.20221018202458-cffd2bb71c7b h1:Ha+kSIoTutf4ytlVw/SaEclDUloYx0+FXDKJWKhNbE4=
 github.com/grafana/e2e v0.1.1-0.20221018202458-cffd2bb71c7b/go.mod h1:3UsooRp7yW5/NJQBlXcTsAHOoykEhNUYXkQ3r6ehEEY=
 github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9 h1:WB3bGH2f1UN6jkd6uAEWfHB8OD7dKJ0v2Oo6SNfhpfQ=
@@ -1321,8 +1321,6 @@ github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
 github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=
-github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=
-github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
 github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=

integration/e2e/e2e_integration_test1032424306/config.yaml

@@ -0,0 +1,45 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    jaeger:
+      protocols:
+        grpc:
+
+ingester:
+  lifecycler:
+    ring:
+      replication_factor: 3
+    heartbeat_period: 100ms
+  override_ring_key: "ring-a"
+
+storage:
+  trace:
+    backend: s3
+    s3:
+      bucket: tempo
+      endpoint: tempo_active_active-minio-9000:9000  # TODO: this is brittle, fix this eventually
+      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
+      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
+      insecure: true
+    pool:
+      max_workers: 10
+      queue_depth: 100
+
+memberlist:
+  abort_if_cluster_join_fails: false
+  bind_port: 7946
+  join_members:
+  - tempo_active_active-ingester-a-1:7946
+  - tempo_active_active-ingester-a-2:7946
+  - tempo_active_active-ingester-a-3:7946
+  - tempo_active_active-ingester-b-1:7946
+  - tempo_active_active-ingester-b-2:7946
+  - tempo_active_active-ingester-b-3:7946
+
+
+querier:
+  secondary_ingester_ring: "ring-b"
+  frontend_worker:
+    frontend_address: tempo_active_active-query-frontend-a:9095

modules/distributor/distributor.go

@@ -227,7 +227,7 @@ func New(cfg Config, clientCfg ingester_client.Config, ingestersRing ring.ReadRi
 		cfgReceivers = defaultReceivers
 	}
 
-	receivers, err := receiver.New(cfgReceivers, d, middleware, loggingLevel)
+	receivers, err := receiver.New(cfgReceivers, d, middleware, loggingLevel, reg)
 	if err != nil {
 		return nil, err
 	}

modules/distributor/forwarder/forwarder.go

@@ -12,7 +12,6 @@ import (
 	zaplogfmt "github.com/jsternberg/zap-logfmt"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor"
-	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/extension"
@@ -191,23 +190,23 @@ func (c consumerToForwarderAdapter) Capabilities() consumer.Capabilities {
 	return consumer.Capabilities{MutatesData: false}
 }
 
-func newLogger(level dslog.Level) *zap.Logger {
-	zapLevel := zapcore.InfoLevel
-
-	switch level.Logrus {
-	case logrus.PanicLevel:
-		zapLevel = zapcore.PanicLevel
-	case logrus.FatalLevel:
-		zapLevel = zapcore.FatalLevel
-	case logrus.ErrorLevel:
-		zapLevel = zapcore.ErrorLevel
-	case logrus.WarnLevel:
-		zapLevel = zapcore.WarnLevel
-	case logrus.InfoLevel:
-		zapLevel = zapcore.InfoLevel
-	case logrus.TraceLevel, logrus.DebugLevel:
-		zapLevel = zapcore.DebugLevel
+func ZapLevel(level dslog.Level) zapcore.Level {
+	switch level.String() {
+	case "error":
+		return zapcore.ErrorLevel
+	case "warn":
+		return zapcore.WarnLevel
+	case "info":
+		return zapcore.InfoLevel
+	case "debug":
+		return zapcore.DebugLevel
+	default:
+		return zapcore.InfoLevel
 	}
+}
+
+func newLogger(level dslog.Level) *zap.Logger {
+	zapLevel := ZapLevel(level)
 
 	config := zap.NewProductionEncoderConfig()
 	config.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {

modules/distributor/receiver/shim.go

@@ -6,9 +6,11 @@ import (
 	"os"
 	"time"
 
+	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	dslog "github.com/grafana/dskit/log"
 	"github.com/grafana/dskit/services"
+	"github.com/grafana/tempo/modules/distributor/forwarder"
 	zaplogfmt "github.com/jsternberg/zap-logfmt"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver"
@@ -17,7 +19,6 @@ import (
 	"github.com/opentracing/opentracing-go"
 	prom_client "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/sirupsen/logrus"
 	"go.opencensus.io/stats/view"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/confmap"
@@ -57,6 +58,8 @@ var (
 	statReceiverZipkin     = usagestats.NewInt("receiver_enabled_zipkin")
 	statReceiverOpencensus = usagestats.NewInt("receiver_enabled_opencensus")
 	statReceiverKafka      = usagestats.NewInt("receiver_enabled_kafka")
+
+	rateLimitedLogger kitlog.Logger
 )
 
 type TracesPusher interface {
@@ -70,7 +73,7 @@ type receiversShim struct {
 
 	receivers   []receiver.Traces
 	pusher      TracesPusher
-	logger      *log.RateLimitedLogger
+	logger      kitlog.Logger
 	metricViews []*view.View
 	fatal       chan error
 }
@@ -94,10 +97,20 @@ func (m *mapProvider) Scheme() string { return "mock" }
 
 func (m *mapProvider) Shutdown(context.Context) error { return nil }
 
-func New(receiverCfg map[string]interface{}, pusher TracesPusher, middleware Middleware, logLevel dslog.Level) (services.Service, error) {
+func getRateLimitedLogger(registry prom_client.Registerer) kitlog.Logger {
+	if rateLimitedLogger == nil {
+		rateLimitedLogger = dslog.NewRateLimitedLogger(log.Logger, logsPerSecond, 1, registry)
+		rateLimitedLogger = level.Error(rateLimitedLogger)
+	}
+	return rateLimitedLogger
+}
+
+func New(receiverCfg map[string]interface{}, pusher TracesPusher, middleware Middleware, logLevel dslog.Level, registry prom_client.Registerer) (services.Service, error) {
+	rateLimitedLogger := getRateLimitedLogger(registry)
+
 	shim := &receiversShim{
 		pusher: pusher,
-		logger: log.NewRateLimitedLogger(logsPerSecond, level.Error(log.Logger)),
+		logger: rateLimitedLogger,
 		fatal:  make(chan error),
 	}
 
@@ -313,23 +326,7 @@ func (r *receiversShim) GetExporters() map[component.DataType]map[component.ID]c
 
 // observability shims
 func newLogger(level dslog.Level) *zap.Logger {
-	zapLevel := zapcore.InfoLevel
-
-	switch level.Logrus {
-	case logrus.PanicLevel:
-		zapLevel = zapcore.PanicLevel
-	case logrus.FatalLevel:
-		zapLevel = zapcore.FatalLevel
-	case logrus.ErrorLevel:
-		zapLevel = zapcore.ErrorLevel
-	case logrus.WarnLevel:
-		zapLevel = zapcore.WarnLevel
-	case logrus.InfoLevel:
-		zapLevel = zapcore.InfoLevel
-	case logrus.DebugLevel:
-	case logrus.TraceLevel:
-		zapLevel = zapcore.DebugLevel
-	}
+	zapLevel := forwarder.ZapLevel(level)
 
 	config := zap.NewProductionEncoderConfig()
 	config.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {

pkg/util/log/log.go

@@ -6,51 +6,24 @@ import (
 	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	dslog "github.com/grafana/dskit/log"
-	"github.com/grafana/dskit/server"
 )
 
 // Logger is a shared go-kit logger.
 // TODO: Change all components to take a non-global logger via their constructors.
 // Prefer accepting a non-global logger as an argument.
 var Logger = kitlog.NewNopLogger()
 
-// InitLogger initialises the global gokit logger and overrides the
-// default logger for the server.
-func InitLogger(cfg *server.Config) {
-	logger := kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stderr))
-	if cfg.LogFormat.String() == "json" {
-		logger = kitlog.NewJSONLogger(kitlog.NewSyncWriter(os.Stderr))
-	}
+// InitLogger initialises the global gokit logger and returns that logger.
+func InitLogger(logFormat string, logLevel dslog.Level) kitlog.Logger {
+	writer := kitlog.NewSyncWriter(os.Stderr)
+	logger := dslog.NewGoKitWithWriter(logFormat, writer)
 
-	// add support for level based logging
-	logger = level.NewFilter(logger, LevelFilter(cfg.LogLevel.String()))
-
-	// use UTC timestamps
+	// use UTC timestamps and skip 3 stack frames.
 	logger = kitlog.With(logger, "ts", kitlog.DefaultTimestampUTC)
 
-	// when use util_log.Logger, skip 3 stack frames.
-	Logger = kitlog.With(logger, "caller", kitlog.Caller(3))
-
-	// cfg.Log wraps log function, skip 4 stack frames to get caller information.
-	// this works in go 1.12, but doesn't work in versions earlier.
-	// it will always shows the wrapper function generated by compiler
-	// marked <autogenerated> in old versions.
-	cfg.Log = dslog.GoKit(kitlog.With(logger, "caller", kitlog.Caller(4)))
-}
+	// Must put the level filter last for efficiency.
+	logger = level.NewFilter(logger, logLevel.Option)
 
-// TODO: remove once weaveworks/common updates to go-kit/log
-// -> we can then revert to using Level.Gokit
-func LevelFilter(l string) level.Option {
-	switch l {
-	case "debug":
-		return level.AllowDebug()
-	case "info":
-		return level.AllowInfo()
-	case "warn":
-		return level.AllowWarn()
-	case "error":
-		return level.AllowError()
-	default:
-		return level.AllowAll()
-	}
+	Logger = logger
+	return logger
 }