diff --git a/CHANGELOG.md b/CHANGELOG.md index 7347068756d4..1462f67289f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ Changes by Version ================== -1.20.0 (unreleased) +1.21.0 (unreleased) +------------------- + +1.20.0 (2020-09-29) ------------------- ### Backend Changes @@ -26,10 +29,47 @@ Changes by Version #### New Features +* Grpc plugin archive storage support ([#2317](https://github.com/jaegertracing/jaeger/pull/2317), [@m8rge](https://github.com/m8rge)) +* Separate Ports for GRPC and HTTP requests in Query Server ([#2387](https://github.com/jaegertracing/jaeger/pull/2387), [@rjs211](https://github.com/rjs211)) +* Configurable ES doc count ([#2453](https://github.com/jaegertracing/jaeger/pull/2453), [@albertteoh](https://github.com/albertteoh)) +* Add storage metrics to OTEL, metrics by span service name ([#2431](https://github.com/jaegertracing/jaeger/pull/2431), [@pavolloffay](https://github.com/pavolloffay)) + #### Bug fixes, Minor Improvements +* Increase coverage on otel/app/defaultconfig and otel/app/defaultcomponents ([#2515](https://github.com/jaegertracing/jaeger/pull/2515), [@joe-elliott](https://github.com/joe-elliott)) +* Use OTEL Kafka Exporter/Receiver Instead of Jaeger Core ([#2494](https://github.com/jaegertracing/jaeger/pull/2494), [@joe-elliott](https://github.com/joe-elliott)) +* Fix OTEL kafka receiver/ingester panic ([#2512](https://github.com/jaegertracing/jaeger/pull/2512), [@pavolloffay](https://github.com/pavolloffay)) +* Disable clock-skew-adjustment by default. ([#2513](https://github.com/jaegertracing/jaeger/pull/2513), [@jpkrohling](https://github.com/jpkrohling)) +* Fix ES OTEL status code ([#2501](https://github.com/jaegertracing/jaeger/pull/2501), [@pavolloffay](https://github.com/pavolloffay)) +* OTel: Factored out Config Factory ([#2495](https://github.com/jaegertracing/jaeger/pull/2495), [@joe-elliott](https://github.com/joe-elliott)) +* Fix failing ServerInUseHostPort test on MacOS ([#2477](https://github.com/jaegertracing/jaeger/pull/2477), [@albertteoh](https://github.com/albertteoh)) +* Fix unmarshalling in OTEL badger ([#2488](https://github.com/jaegertracing/jaeger/pull/2488), [@pavolloffay](https://github.com/pavolloffay)) +* Improve UI placeholder message ([#2487](https://github.com/jaegertracing/jaeger/pull/2487), [@yurishkuro](https://github.com/yurishkuro)) +* Translate OTEL instrumentation library to ES DB model ([#2484](https://github.com/jaegertracing/jaeger/pull/2484), [@pavolloffay](https://github.com/pavolloffay)) +* Add partial retry capability to OTEL ES exporter. ([#2456](https://github.com/jaegertracing/jaeger/pull/2456), [@pavolloffay](https://github.com/pavolloffay)) +* Log deprecation warning only when deprecated flags are set ([#2479](https://github.com/jaegertracing/jaeger/pull/2479), [@pavolloffay](https://github.com/pavolloffay)) +* Clean-up Badger's trace-not-found check ([#2481](https://github.com/jaegertracing/jaeger/pull/2481), [@yurishkuro](https://github.com/yurishkuro)) +* Run the jaeger-agent as a non-root user by default ([#2466](https://github.com/jaegertracing/jaeger/pull/2466), [@chgl](https://github.com/chgl)) +* Regenerate certificates to use SANs instead of Common Name ([#2461](https://github.com/jaegertracing/jaeger/pull/2461), [@albertteoh](https://github.com/albertteoh)) +* Support custom port in cassandra schema creation ([#2472](https://github.com/jaegertracing/jaeger/pull/2472), [@MarianZoll](https://github.com/MarianZoll)) +* Consolidated OTel ES IndexNameProviders ([#2458](https://github.com/jaegertracing/jaeger/pull/2458), [@joe-elliott](https://github.com/joe-elliott)) +* Add positive confirmation that Agent made a connection to Collector (… ([#2423](https://github.com/jaegertracing/jaeger/pull/2423), [@BernardTolosajr](https://github.com/BernardTolosajr)) +* Propagate TraceNotFound error from grpc storage plugins ([#2455](https://github.com/jaegertracing/jaeger/pull/2455), [@joe-elliott](https://github.com/joe-elliott)) +* Use new ES reader implementation in OTEL ([#2441](https://github.com/jaegertracing/jaeger/pull/2441), [@pavolloffay](https://github.com/pavolloffay)) +* Updated grpc-go to v1.29.1 ([#2445](https://github.com/jaegertracing/jaeger/pull/2445), [@jpkrohling](https://github.com/jpkrohling)) +* Remove olivere elastic client from OTEL ([#2448](https://github.com/jaegertracing/jaeger/pull/2448), [@pavolloffay](https://github.com/pavolloffay)) +* Use queue retry per exporter ([#2444](https://github.com/jaegertracing/jaeger/pull/2444), [@pavolloffay](https://github.com/pavolloffay)) +* Add context.Context to WriteSpan ([#2436](https://github.com/jaegertracing/jaeger/pull/2436), [@yurishkuro](https://github.com/yurishkuro)) +* Fix mutex unlock in storage exporters ([#2442](https://github.com/jaegertracing/jaeger/pull/2442), [@pavolloffay](https://github.com/pavolloffay)) +* Add Grafana integration example ([#2408](https://github.com/jaegertracing/jaeger/pull/2408), [@fktkrt](https://github.com/fktkrt)) +* Fix TLS flags settings in jaeger OTEL receiver ([#2438](https://github.com/jaegertracing/jaeger/pull/2438), [@pavolloffay](https://github.com/pavolloffay)) +* Add context to dependencies endpoint ([#2434](https://github.com/jaegertracing/jaeger/pull/2434), [@yoave23](https://github.com/yoave23)) +* Fix error equals ([#2429](https://github.com/jaegertracing/jaeger/pull/2429), [@albertteoh](https://github.com/albertteoh)) + ### UI Changes +* UI pinned to version 1.11.0. The changelog is available here [v1.11.0](https://github.com/jaegertracing/jaeger-ui/blob/master/CHANGELOG.md#v1110-september-28-2020) + 1.19.2 (2020-08-26) ------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 720d83f5db9f..8f562cd56852 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ We gratefully welcome improvements to documentation as well as to code. ## Getting Started ### Pre-requisites -* Install [Go](https://golang.org/doc/install) and setup GOPATH and add $GOPATH/bin in PATH +* Install [Go](https://golang.org/doc/install) and setup GOPATH and add $GOPATH/bin in PATH This library uses Go modules to manage dependencies. @@ -116,9 +116,19 @@ import ( ## Testing guidelines -We strive to maintain as high code coverage as possible. Since `go test` command does not generate +We strive to maintain as high code coverage as possible. The current repository limit is set at 95%, +with some exclusions discussed below. + +### Combining code coverage + +We use [cover.sh](./scripts/cover.sh) script to run tests and combine code coverage from all packages +(see also [issue # 797](https://github.com/jaegertracing/jaeger/issues/797)). + +### Packages with no tests + +Since `go test` command does not generate code coverage information for packages that have no test files, we have a build step (`make nocover`) -that breaks the build when such packages are discovered, with an error like this: +that breaks the build when such packages are discovered, with the following error: ``` error: at least one *_test.go file must be in all directories with go files @@ -126,8 +136,12 @@ error: at least one *_test.go file must be in all directories with go files If no tests are possible for a package (e.g. it only defines types), create empty_test.go ``` +As the message says, all packages are required to have at least one `*_test.go` file. + +### Excluding packages from testing + There are conditions that cannot be tested without external dependencies, such as a function that -creates a gocql.Session, because it requires an active connection to Cassandra database. It is +creates a `gocql.Session`, because it requires an active connection to Cassandra database. It is recommended to isolate such functions in a separate package with bare minimum of code and add a file `.nocover` to exclude the package from coverage calculations. The file should contain a comment explaining why it is there, for example: @@ -145,4 +159,3 @@ Before merging a PR make sure: Merge the PR by using "Squash and merge" option on Github. Avoid creating merge commits. After the merge make sure referenced issues were closed. - diff --git a/README.md b/README.md index 39179bf49ab8..74dbea2954c5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1273/badge)](https://bestpractices.coreinfrastructure.org/projects/1273) -[![Go Report Card](https://goreportcard.com/badge/github.com/jaegertracing/jaeger?style=flat-square)](https://goreportcard.com/report/github.com/jaegertracing/jaeger) [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go#performance) [![OpenTracing-1.0][ot-badge]](https://opentracing.io) @@ -110,8 +109,7 @@ of routing the traffic from Zipkin libraries to the Jaeger backend. ### Deployment - * [Kubernetes templates](https://github.com/jaegertracing/jaeger-kubernetes) - * [OpenShift templates](https://github.com/jaegertracing/jaeger-openshift) + * [Jaeger Operator for Kubernetes](https://github.com/jaegertracing/jaeger-operator#getting-started) ### Components diff --git a/cmd/agent/app/agent_test.go b/cmd/agent/app/agent_test.go index 52a092c51b52..b31346feb53c 100644 --- a/cmd/agent/app/agent_test.go +++ b/cmd/agent/app/agent_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber/jaeger-lib/metrics" + "github.com/uber/jaeger-lib/metrics/fork" "go.uber.org/zap" jmetrics "github.com/jaegertracing/jaeger/pkg/metrics" @@ -100,7 +101,8 @@ func withRunningAgent(t *testing.T, testcase func(string, chan error)) { } logger, logBuf := testutils.NewLogger() mBldr := &jmetrics.Builder{HTTPRoute: "/metrics", Backend: "prometheus"} - mFactory, err := mBldr.CreateMetricsFactory("jaeger") + metricsFactory, err := mBldr.CreateMetricsFactory("jaeger") + mFactory := fork.New("internal", metrics.NullFactory, metricsFactory) require.NoError(t, err) agent, err := cfg.CreateAgent(fakeCollectorProxy{}, logger, mFactory) require.NoError(t, err) @@ -162,7 +164,8 @@ func TestStartStopRace(t *testing.T) { } logger, logBuf := testutils.NewLogger() mBldr := &jmetrics.Builder{HTTPRoute: "/metrics", Backend: "prometheus"} - mFactory, err := mBldr.CreateMetricsFactory("jaeger") + metricsFactory, err := mBldr.CreateMetricsFactory("jaeger") + mFactory := fork.New("internal", metrics.NullFactory, metricsFactory) require.NoError(t, err) agent, err := cfg.CreateAgent(fakeCollectorProxy{}, logger, mFactory) require.NoError(t, err) diff --git a/cmd/agent/app/builder.go b/cmd/agent/app/builder.go index 09732e6b839c..c77d5c6165fc 100644 --- a/cmd/agent/app/builder.go +++ b/cmd/agent/app/builder.go @@ -112,6 +112,8 @@ func (b *Builder) CreateAgent(primaryProxy CollectorProxy, logger *zap.Logger, m return nil, fmt.Errorf("cannot create processors: %w", err) } server := b.HTTPServer.getHTTPServer(primaryProxy.GetManager(), mFactory) + b.publishOpts(mFactory) + return NewAgent(processors, server, logger), nil } @@ -127,6 +129,19 @@ func (b *Builder) getReporter(primaryProxy CollectorProxy) reporter.Reporter { return reporter.NewMultiReporter(rep...) } +func (b *Builder) publishOpts(mFactory metrics.Factory) { + internalFactory := mFactory.Namespace(metrics.NSOptions{Name: "internal"}) + for _, p := range b.Processors { + prefix := fmt.Sprintf(processorPrefixFmt, p.Model, p.Protocol) + internalFactory.Gauge(metrics.Options{Name: prefix + suffixServerMaxPacketSize}). + Update(int64(p.Server.MaxPacketSize)) + internalFactory.Gauge(metrics.Options{Name: prefix + suffixServerQueueSize}). + Update(int64(p.Server.QueueSize)) + internalFactory.Gauge(metrics.Options{Name: prefix + suffixWorkers}). + Update(int64(p.Workers)) + } +} + func (b *Builder) getProcessors(rep reporter.Reporter, mFactory metrics.Factory, logger *zap.Logger) ([]processors.Processor, error) { retMe := make([]processors.Processor, len(b.Processors)) for idx, cfg := range b.Processors { diff --git a/cmd/agent/app/builder_test.go b/cmd/agent/app/builder_test.go index 7d2dd74500d4..49c2e9d58be3 100644 --- a/cmd/agent/app/builder_test.go +++ b/cmd/agent/app/builder_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber/jaeger-lib/metrics" + "github.com/uber/jaeger-lib/metrics/fork" "github.com/uber/jaeger-lib/metrics/metricstest" "go.uber.org/zap" yaml "gopkg.in/yaml.v2" @@ -277,3 +278,42 @@ func TestCreateCollectorProxy_UnknownReporter(t *testing.T) { assert.Nil(t, proxy) assert.EqualError(t, err, "unknown reporter type ") } + +func TestPublishOpts(t *testing.T) { + v := viper.New() + cfg := &Builder{} + command := cobra.Command{} + flags := &flag.FlagSet{} + AddFlags(flags) + command.PersistentFlags().AddGoFlagSet(flags) + v.BindPFlags(command.PersistentFlags()) + err := command.ParseFlags([]string{ + "--http-server.host-port=:8080", + "--processor.jaeger-binary.server-host-port=:1111", + "--processor.jaeger-binary.server-max-packet-size=4242", + "--processor.jaeger-binary.server-queue-size=24", + "--processor.jaeger-binary.workers=42", + }) + require.NoError(t, err) + cfg.InitFromViper(v) + + baseMetrics := metricstest.NewFactory(time.Second) + forkFactory := metricstest.NewFactory(time.Second) + metricsFactory := fork.New("internal", forkFactory, baseMetrics) + agent, err := cfg.CreateAgent(fakeCollectorProxy{}, zap.NewNop(), metricsFactory) + assert.NoError(t, err) + assert.NotNil(t, agent) + + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal.processor.jaeger-binary.server-max-packet-size", + Value: 4242, + }) + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal.processor.jaeger-binary.server-queue-size", + Value: 24, + }) + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal.processor.jaeger-binary.workers", + Value: 42, + }) +} diff --git a/cmd/agent/app/flags.go b/cmd/agent/app/flags.go index f0fd4e030632..28f22e6cbac2 100644 --- a/cmd/agent/app/flags.go +++ b/cmd/agent/app/flags.go @@ -31,6 +31,9 @@ const ( suffixServerMaxPacketSize = "server-max-packet-size" suffixServerSocketBufferSize = "server-socket-buffer-size" suffixServerHostPort = "server-host-port" + + processorPrefixFmt = "processor.%s-%s." + // HTTPServerHostPort is the flag for HTTP endpoint HTTPServerHostPort = "http-server.host-port" ) @@ -48,7 +51,7 @@ var defaultProcessors = []struct { // AddFlags adds flags for Builder. func AddFlags(flags *flag.FlagSet) { for _, p := range defaultProcessors { - prefix := fmt.Sprintf("processor.%s-%s.", p.model, p.protocol) + prefix := fmt.Sprintf(processorPrefixFmt, p.model, p.protocol) flags.Int(prefix+suffixWorkers, defaultServerWorkers, "how many workers the processor should run") flags.Int(prefix+suffixServerQueueSize, defaultQueueSize, "length of the queue for the UDP server") flags.Int(prefix+suffixServerMaxPacketSize, defaultMaxPacketSize, "max packet size for the UDP server") @@ -69,7 +72,7 @@ func AddOTELFlags(flags *flag.FlagSet) { // InitFromViper initializes Builder with properties retrieved from Viper. func (b *Builder) InitFromViper(v *viper.Viper) *Builder { for _, processor := range defaultProcessors { - prefix := fmt.Sprintf("processor.%s-%s.", processor.model, processor.protocol) + prefix := fmt.Sprintf(processorPrefixFmt, processor.model, processor.protocol) p := &ProcessorConfiguration{Model: processor.model, Protocol: processor.protocol} p.Workers = v.GetInt(prefix + suffixWorkers) p.Server.QueueSize = v.GetInt(prefix + suffixServerQueueSize) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 7615d00ec795..c01d04d3ea75 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -22,6 +22,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/uber/jaeger-lib/metrics" + jexpvar "github.com/uber/jaeger-lib/metrics/expvar" + "github.com/uber/jaeger-lib/metrics/fork" _ "go.uber.org/automaxprocs" "go.uber.org/zap" @@ -49,9 +51,12 @@ func main() { return err } logger := svc.Logger // shortcut - mFactory := svc.MetricsFactory. + baseFactory := svc.MetricsFactory. Namespace(metrics.NSOptions{Name: "jaeger"}). Namespace(metrics.NSOptions{Name: "agent"}) + mFactory := fork.New("internal", + jexpvar.NewFactory(10), // backend for internal opts + baseFactory) rOpts := new(reporter.Options).InitFromViper(v, logger) grpcBuilder := grpc.NewConnBuilder().InitFromViper(v) @@ -79,6 +84,7 @@ func main() { if err := agent.Run(); err != nil { return fmt.Errorf("failed to run the agent: %w", err) } + svc.RunAndThen(func() { agent.Stop() cp.Close() diff --git a/cmd/all-in-one/main.go b/cmd/all-in-one/main.go index 864d7e1d974e..cc600ce58666 100644 --- a/cmd/all-in-one/main.go +++ b/cmd/all-in-one/main.go @@ -27,6 +27,8 @@ import ( jaegerClientConfig "github.com/uber/jaeger-client-go/config" jaegerClientZapLog "github.com/uber/jaeger-client-go/log/zap" "github.com/uber/jaeger-lib/metrics" + jexpvar "github.com/uber/jaeger-lib/metrics/expvar" + "github.com/uber/jaeger-lib/metrics/fork" _ "go.uber.org/automaxprocs" "go.uber.org/zap" @@ -81,7 +83,10 @@ by default uses only in-memory database.`, } logger := svc.Logger // shortcut rootMetricsFactory := svc.MetricsFactory // shortcut - metricsFactory := rootMetricsFactory.Namespace(metrics.NSOptions{Name: "jaeger"}) + metricsFactory := fork.New("internal", + jexpvar.NewFactory(10), // backend for internal opts + rootMetricsFactory.Namespace(metrics.NSOptions{Name: "jaeger"})) + tracerCloser := initTracer(rootMetricsFactory, svc.Logger) storageFactory.InitFromViper(v) diff --git a/cmd/collector/app/builder_flags.go b/cmd/collector/app/builder_flags.go index 1bb9a4e760e3..82e4df5ae32f 100644 --- a/cmd/collector/app/builder_flags.go +++ b/cmd/collector/app/builder_flags.go @@ -116,5 +116,6 @@ func (cOpts *CollectorOptions) InitFromViper(v *viper.Viper) *CollectorOptions { cOpts.CollectorZipkinAllowedOrigins = v.GetString(collectorZipkinAllowedOrigins) cOpts.CollectorZipkinAllowedHeaders = v.GetString(collectorZipkinAllowedHeaders) cOpts.TLS = tlsFlagsConfig.InitFromViper(v) + return cOpts } diff --git a/cmd/collector/app/collector.go b/cmd/collector/app/collector.go index 9f08e285b538..fc6e53c1dd68 100644 --- a/cmd/collector/app/collector.go +++ b/cmd/collector/app/collector.go @@ -122,10 +122,17 @@ func (c *Collector) Start(builderOpts *CollectorOptions) error { } else { c.zkServer = zkServer } + c.publishOpts(builderOpts) return nil } +func (c *Collector) publishOpts(cOpts *CollectorOptions) { + internalFactory := c.metricsFactory.Namespace(metrics.NSOptions{Name: "internal"}) + internalFactory.Gauge(metrics.Options{Name: collectorNumWorkers}).Update(int64(cOpts.NumWorkers)) + internalFactory.Gauge(metrics.Options{Name: collectorQueueSize}).Update(int64(cOpts.QueueSize)) +} + // Close the component and all its underlying dependencies func (c *Collector) Close() error { // gRPC server diff --git a/cmd/collector/app/collector_test.go b/cmd/collector/app/collector_test.go index 98f876af9d36..9686177f9974 100644 --- a/cmd/collector/app/collector_test.go +++ b/cmd/collector/app/collector_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/uber/jaeger-lib/metrics/fork" "github.com/uber/jaeger-lib/metrics/metricstest" "go.uber.org/zap" @@ -61,3 +62,39 @@ type mockStrategyStore struct { func (m *mockStrategyStore) GetSamplingStrategy(_ context.Context, serviceName string) (*sampling.SamplingStrategyResponse, error) { return &sampling.SamplingStrategyResponse{}, nil } + +func TestCollector_PublishOpts(t *testing.T) { + // prepare + hc := healthcheck.New() + logger := zap.NewNop() + baseMetrics := metricstest.NewFactory(time.Second) + forkFactory := metricstest.NewFactory(time.Second) + metricsFactory := fork.New("internal", forkFactory, baseMetrics) + spanWriter := &fakeSpanWriter{} + strategyStore := &mockStrategyStore{} + + c := New(&CollectorParams{ + ServiceName: "collector", + Logger: logger, + MetricsFactory: metricsFactory, + SpanWriter: spanWriter, + StrategyStore: strategyStore, + HealthCheck: hc, + }) + collectorOpts := &CollectorOptions{ + NumWorkers: 24, + QueueSize: 42, + } + + c.Start(collectorOpts) + defer c.Close() + + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal.collector.num-workers", + Value: 24, + }) + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal.collector.queue-size", + Value: 42, + }) +} diff --git a/cmd/collector/main.go b/cmd/collector/main.go index 1d42f4e6d37e..976ede793747 100644 --- a/cmd/collector/main.go +++ b/cmd/collector/main.go @@ -24,6 +24,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/uber/jaeger-lib/metrics" + jexpvar "github.com/uber/jaeger-lib/metrics/expvar" + "github.com/uber/jaeger-lib/metrics/fork" _ "go.uber.org/automaxprocs" "go.uber.org/zap" @@ -63,7 +65,9 @@ func main() { } logger := svc.Logger // shortcut baseFactory := svc.MetricsFactory.Namespace(metrics.NSOptions{Name: "jaeger"}) - metricsFactory := baseFactory.Namespace(metrics.NSOptions{Name: "collector"}) + metricsFactory := fork.New("internal", + jexpvar.NewFactory(10), // backend for internal opts + baseFactory.Namespace(metrics.NSOptions{Name: "collector"})) storageFactory.InitFromViper(v) if err := storageFactory.Initialize(baseFactory, logger); err != nil { diff --git a/cmd/flags/service.go b/cmd/flags/service.go index d6b02cf3a53b..0b053cbc4d42 100644 --- a/cmd/flags/service.go +++ b/cmd/flags/service.go @@ -15,6 +15,7 @@ package flags import ( + "expvar" "flag" "fmt" "os" @@ -116,6 +117,13 @@ func (s *Service) Start(v *viper.Viper) error { s.Logger.Info("Mounting metrics handler on admin server", zap.String("route", route)) s.Admin.Handle(route, h) } + + // Mount expvar routes on different backends + if metricsBuilder.Backend != "expvar" { + s.Logger.Info("Mounting expvar handler on admin server", zap.String("route", "/debug/vars")) + s.Admin.Handle("/debug/vars", expvar.Handler()) + } + if err := s.Admin.Serve(); err != nil { return fmt.Errorf("cannot start the admin server: %w", err) } diff --git a/cmd/opentelemetry/app/defaultcomponents/defaults_test.go b/cmd/opentelemetry/app/defaultcomponents/defaults_test.go index 950dc782b514..61437f49a074 100644 --- a/cmd/opentelemetry/app/defaultcomponents/defaults_test.go +++ b/cmd/opentelemetry/app/defaultcomponents/defaults_test.go @@ -52,7 +52,16 @@ func TestComponents(t *testing.T) { cassandraFactory := factories.Exporters[cassandraexporter.TypeStr] cc := cassandraFactory.CreateDefaultConfig().(*cassandraexporter.Config) assert.Equal(t, []string{"127.0.0.1"}, cc.Options.GetPrimary().Servers) + esFactory := factories.Exporters[elasticsearchexporter.TypeStr] ec := esFactory.CreateDefaultConfig().(*elasticsearchexporter.Config) assert.Equal(t, []string{"http://127.0.0.1:9200"}, ec.GetPrimary().Servers) + + grpcFactory := factories.Exporters[grpcpluginexporter.TypeStr] + gc := grpcFactory.CreateDefaultConfig().(*grpcpluginexporter.Config) + assert.Equal(t, "", gc.Configuration.PluginBinary) + + badgerFactory := factories.Exporters[badgerexporter.TypeStr] + bc := badgerFactory.CreateDefaultConfig().(*badgerexporter.Config) + assert.Equal(t, "", bc.GetPrimary().ValueDirectory) } diff --git a/cmd/opentelemetry/app/defaultconfig/default_config_test.go b/cmd/opentelemetry/app/defaultconfig/default_config_test.go index 888da5c3e664..21551d943546 100644 --- a/cmd/opentelemetry/app/defaultconfig/default_config_test.go +++ b/cmd/opentelemetry/app/defaultconfig/default_config_test.go @@ -15,14 +15,17 @@ package defaultconfig import ( + "flag" "fmt" "sort" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/configmodels" + "go.opentelemetry.io/collector/service/builder" "go.uber.org/zap" "github.com/jaegertracing/jaeger/cmd/opentelemetry/app" @@ -42,6 +45,7 @@ func TestService(t *testing.T) { cfg ComponentSettings err string viperConfig map[string]interface{} + otelConfig string }{ { cfg: ComponentSettings{ @@ -59,6 +63,41 @@ func TestService(t *testing.T) { }, }, }, + { + cfg: ComponentSettings{ + ComponentType: Collector, + StorageType: "badger", + }, + service: configmodels.Service{ + Extensions: []string{"health_check"}, + Pipelines: configmodels.Pipelines{ + "traces": &configmodels.Pipeline{ + InputType: configmodels.TracesDataType, + Receivers: []string{"otlp", "jaeger"}, + Processors: []string{"batch"}, + Exporters: []string{"jaeger_badger"}, + }, + }, + }, + }, + { + cfg: ComponentSettings{ + ComponentType: Agent, + }, + service: configmodels.Service{ + Extensions: []string{"health_check"}, + Pipelines: configmodels.Pipelines{ + "traces": &configmodels.Pipeline{ + Name: "traces", + InputType: configmodels.TracesDataType, + Receivers: []string{"otlp", "jaeger"}, + Processors: []string{"batch", "queued_retry"}, + Exporters: []string{"jaeger"}, + }, + }, + }, + otelConfig: "testdata/addqueuedprocessor.yaml", + }, { viperConfig: map[string]interface{}{"resource.attributes": "foo=bar"}, cfg: ComponentSettings{ @@ -136,6 +175,13 @@ func TestService(t *testing.T) { }, err: "unknown storage type: floppy", }, + { + cfg: ComponentSettings{ + ComponentType: Agent, + }, + otelConfig: "testdata/doesntexist.yaml", + err: `error loading config file "testdata/doesntexist.yaml": open testdata/doesntexist.yaml: no such file or directory`, + }, } for _, test := range tests { t.Run(fmt.Sprintf("%v:%v", test.cfg.ComponentType, test.cfg.StorageType), func(t *testing.T) { @@ -143,9 +189,18 @@ func TestService(t *testing.T) { for key, val := range test.viperConfig { v.Set(key, val) } + + otelFlags := &flag.FlagSet{} + builder.Flags(otelFlags) + if test.otelConfig != "" { + otelFlags.Parse([]string{"--config=" + test.otelConfig}) + } + factories := defaultcomponents.Components(v) test.cfg.Factories = factories - cfg, err := test.cfg.createDefaultConfig() + createDefaultConfig := test.cfg.DefaultConfigFactory(v) + + cfg, err := createDefaultConfig(viper.New(), factories) if test.err != "" { require.Nil(t, cfg) assert.Contains(t, err.Error(), test.err) diff --git a/cmd/opentelemetry/app/defaultconfig/merge.go b/cmd/opentelemetry/app/defaultconfig/merge.go index 1a057f12b273..cc19ab8b3cd3 100644 --- a/cmd/opentelemetry/app/defaultconfig/merge.go +++ b/cmd/opentelemetry/app/defaultconfig/merge.go @@ -25,10 +25,5 @@ func MergeConfigs(dst, src *configmodels.Config) error { if src == nil { return nil } - err := mergo.Merge(dst, src, - mergo.WithOverride) - if err != nil { - return err - } - return nil + return mergo.Merge(dst, src, mergo.WithOverride) } diff --git a/cmd/opentelemetry/app/defaultconfig/testdata/addqueuedprocessor.yaml b/cmd/opentelemetry/app/defaultconfig/testdata/addqueuedprocessor.yaml new file mode 100644 index 000000000000..ccffb43e2842 --- /dev/null +++ b/cmd/opentelemetry/app/defaultconfig/testdata/addqueuedprocessor.yaml @@ -0,0 +1,7 @@ +processors: + queued_retry: + +service: + pipelines: + traces: + processors: [batch, queued_retry] diff --git a/cmd/opentelemetry/app/exporter/cassandraexporter/factory.go b/cmd/opentelemetry/app/exporter/cassandraexporter/factory.go index 3e2ee0953eb9..f9c4f16327ab 100644 --- a/cmd/opentelemetry/app/exporter/cassandraexporter/factory.go +++ b/cmd/opentelemetry/app/exporter/cassandraexporter/factory.go @@ -22,6 +22,7 @@ import ( "go.opentelemetry.io/collector/config/configmodels" "go.opentelemetry.io/collector/exporter/exporterhelper" + collector_app "github.com/jaegertracing/jaeger/cmd/collector/app" "github.com/jaegertracing/jaeger/plugin/storage/cassandra" ) @@ -54,6 +55,10 @@ func (Factory) Type() configmodels.Type { // This function implements OTEL component.ExporterFactoryBase interface. func (f Factory) CreateDefaultConfig() configmodels.Exporter { opts := f.OptionsFactory() + queueSettings := exporterhelper.CreateDefaultQueueSettings() + queueSettings.NumConsumers = collector_app.DefaultNumWorkers + queueSettings.QueueSize = collector_app.DefaultQueueSize + return &Config{ ExporterSettings: configmodels.ExporterSettings{ TypeVal: TypeStr, @@ -61,7 +66,7 @@ func (f Factory) CreateDefaultConfig() configmodels.Exporter { }, TimeoutSettings: exporterhelper.CreateDefaultTimeoutSettings(), RetrySettings: exporterhelper.CreateDefaultRetrySettings(), - QueueSettings: exporterhelper.CreateDefaultQueueSettings(), + QueueSettings: queueSettings, Options: *opts, } } diff --git a/cmd/opentelemetry/app/exporter/cassandraexporter/factory_test.go b/cmd/opentelemetry/app/exporter/cassandraexporter/factory_test.go index ebf055c4262c..7e6bea83f12d 100644 --- a/cmd/opentelemetry/app/exporter/cassandraexporter/factory_test.go +++ b/cmd/opentelemetry/app/exporter/cassandraexporter/factory_test.go @@ -25,6 +25,7 @@ import ( "go.opentelemetry.io/collector/config/configerror" "go.opentelemetry.io/collector/config/configmodels" + collector_app "github.com/jaegertracing/jaeger/cmd/collector/app" jConfig "github.com/jaegertracing/jaeger/pkg/config" "github.com/jaegertracing/jaeger/plugin/storage/cassandra" ) @@ -44,6 +45,9 @@ func TestCreateTraceExporter(t *testing.T) { func TestCreateDefaultConfig(t *testing.T) { factory := Factory{OptionsFactory: DefaultOptions} cfg := factory.CreateDefaultConfig() + + assert.Equal(t, collector_app.DefaultNumWorkers, cfg.(*Config).NumConsumers) + assert.Equal(t, collector_app.DefaultQueueSize, cfg.(*Config).QueueSize) assert.NotNil(t, cfg, "failed to create default config") assert.NoError(t, configcheck.ValidateConfig(cfg)) } diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory.go index 3529d12eb99f..75b9580843f0 100644 --- a/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory.go +++ b/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/collector/config/configmodels" "go.opentelemetry.io/collector/exporter/exporterhelper" + collector_app "github.com/jaegertracing/jaeger/cmd/collector/app" "github.com/jaegertracing/jaeger/plugin/storage/es" ) @@ -54,6 +55,10 @@ var _ component.ExporterFactory = (*Factory)(nil) // CreateDefaultConfig returns default configuration of Factory. // This function implements OTEL component.ExporterFactoryBase interface. func (f Factory) CreateDefaultConfig() configmodels.Exporter { + queueSettings := exporterhelper.CreateDefaultQueueSettings() + queueSettings.NumConsumers = collector_app.DefaultNumWorkers + queueSettings.QueueSize = collector_app.DefaultQueueSize + opts := f.OptionsFactory() return &Config{ ExporterSettings: configmodels.ExporterSettings{ @@ -62,7 +67,7 @@ func (f Factory) CreateDefaultConfig() configmodels.Exporter { }, TimeoutSettings: exporterhelper.CreateDefaultTimeoutSettings(), RetrySettings: exporterhelper.CreateDefaultRetrySettings(), - QueueSettings: exporterhelper.CreateDefaultQueueSettings(), + QueueSettings: queueSettings, Options: *opts, } } diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory_test.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory_test.go index 3ffa169dfd47..e3dec84610ac 100644 --- a/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory_test.go +++ b/cmd/opentelemetry/app/exporter/elasticsearchexporter/factory_test.go @@ -26,6 +26,7 @@ import ( "go.opentelemetry.io/collector/config/configmodels" "go.uber.org/zap" + collector_app "github.com/jaegertracing/jaeger/cmd/collector/app" jConfig "github.com/jaegertracing/jaeger/pkg/config" "github.com/jaegertracing/jaeger/plugin/storage/es" ) @@ -61,6 +62,9 @@ func TestCreateMetricsExporter(t *testing.T) { func TestCreateDefaultConfig(t *testing.T) { factory := Factory{OptionsFactory: DefaultOptions} cfg := factory.CreateDefaultConfig() + assert.Equal(t, collector_app.DefaultNumWorkers, cfg.(*Config).NumConsumers) + assert.Equal(t, collector_app.DefaultQueueSize, cfg.(*Config).QueueSize) + assert.NotNil(t, cfg, "failed to create default config") assert.NoError(t, configcheck.ValidateConfig(cfg)) } diff --git a/cmd/opentelemetry/app/exporter/kafkaexporter/testdata/config.yaml b/cmd/opentelemetry/app/exporter/kafkaexporter/testdata/config.yaml index b0a712554f88..161551e83308 100644 --- a/cmd/opentelemetry/app/exporter/kafkaexporter/testdata/config.yaml +++ b/cmd/opentelemetry/app/exporter/kafkaexporter/testdata/config.yaml @@ -1,31 +1,30 @@ -receivers: - examplereceiver: +receivers: + examplereceiver: -processors: - exampleprocessor: +processors: + exampleprocessor: -exporters: - kafka: - topic: jaeger-prod - encoding: emojis - brokers: foo,bar - auth: - plain_text: - username: user - password: 123 - tls: - ca_file: ca.crt - key_file: key.crt - cert_file: cert.crt - insecure: true - kerberos: - realm: jaeger - config_file: /etc/foo +exporters: + kafka: + topic: jaeger-prod + encoding: emojis + brokers: foo,bar + auth: + plain_text: + username: user + password: 123 + tls: + ca_file: ca.crt + key_file: key.crt + cert_file: cert.crt + insecure: true + kerberos: + realm: jaeger + config_file: /etc/foo - -service: - pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [kafka] \ No newline at end of file +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [kafka] diff --git a/cmd/opentelemetry/app/receiver/kafkareceiver/testdata/config.yaml b/cmd/opentelemetry/app/receiver/kafkareceiver/testdata/config.yaml index 983f26419d78..7bcf02a1597e 100644 --- a/cmd/opentelemetry/app/receiver/kafkareceiver/testdata/config.yaml +++ b/cmd/opentelemetry/app/receiver/kafkareceiver/testdata/config.yaml @@ -1,29 +1,28 @@ -receivers: - kafka: - brokers: foo,bar - topic: jaeger-prod - encoding: emojis - auth: - plain_text: - username: user - password: 123 - tls: - ca_file: ca.crt - key_file: key.crt - insecure: true - kerberos: - config_file: /etc/foo +receivers: + kafka: + brokers: foo,bar + topic: jaeger-prod + encoding: emojis + auth: + plain_text: + username: user + password: 123 + tls: + ca_file: ca.crt + key_file: key.crt + insecure: true + kerberos: + config_file: /etc/foo +processors: + exampleprocessor: -processors: - exampleprocessor: +exporters: + exampleexporter: -exporters: - exampleexporter: - -service: - pipelines: - traces: - receivers: [kafka] - processors: [exampleprocessor] - exporters: [exampleexporter] \ No newline at end of file +service: + pipelines: + traces: + receivers: [kafka] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/cmd/opentelemetry/cmd/agent/.nocover b/cmd/opentelemetry/cmd/agent/.nocover new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cmd/opentelemetry/cmd/all-in-one/.nocover b/cmd/opentelemetry/cmd/all-in-one/.nocover new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cmd/opentelemetry/cmd/collector/.nocover b/cmd/opentelemetry/cmd/collector/.nocover new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cmd/opentelemetry/cmd/ingester/.nocover b/cmd/opentelemetry/cmd/ingester/.nocover new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cmd/opentelemetry/go.mod b/cmd/opentelemetry/go.mod index 8eb93df2802e..f1de0c5f41e3 100644 --- a/cmd/opentelemetry/go.mod +++ b/cmd/opentelemetry/go.mod @@ -15,7 +15,7 @@ require ( github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 github.com/uber/jaeger-client-go v2.25.0+incompatible - github.com/uber/jaeger-lib v2.2.0+incompatible + github.com/uber/jaeger-lib v2.4.0+incompatible go.opencensus.io v0.22.4 go.opentelemetry.io/collector v0.10.1-0.20200917170114-639b9a80ed46 go.uber.org/zap v1.16.0 diff --git a/cmd/opentelemetry/go.sum b/cmd/opentelemetry/go.sum index 2bf928958dd9..bbaf5af1f932 100644 --- a/cmd/opentelemetry/go.sum +++ b/cmd/opentelemetry/go.sum @@ -76,6 +76,8 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk= github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg= +github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -1001,6 +1003,7 @@ github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sectioneight/md-to-godoc v0.0.0-20161108233149-55e43be6c335/go.mod h1:lPZq22klO8la1kyImIDhrGytugMV0TsrsZB55a+xxI0= +github.com/securego/gosec v0.0.0-20200203094520-d13bb6d2420c h1:pThusIwnQVcKbuZSds3HgB/ODEqxMqZf/SgVp89JXY0= github.com/securego/gosec v0.0.0-20200203094520-d13bb6d2420c/go.mod h1:gp0gaHj0WlmPh9BdsTmo1aq6C27yIPWdxCKGFGdVKBE= github.com/securego/gosec/v2 v2.4.0 h1:ivAoWcY5DMs9n04Abc1VkqZBO0FL0h4ShTcVsC53lCE= github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko= @@ -1109,6 +1112,8 @@ github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24sz github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= +github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= @@ -1153,8 +1158,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io v0.1.0 h1:EANZoRCOP+A3faIlw/iN6YEWoYb1vleZRKm1EvH8T48= -go.opentelemetry.io/collector v0.10.0 h1:4T3oARuePrFo8PR6ZYMploL8EcucoaB7tV2hRYg2L+k= go.opentelemetry.io/collector v0.10.1-0.20200917170114-639b9a80ed46 h1:QfvrAwDwB6zp4KuCz6pQxjJaiMbEWWl1Cm/x9Zfnwpk= go.opentelemetry.io/collector v0.10.1-0.20200917170114-639b9a80ed46/go.mod h1:PFIDJqfiYoOFyOZTNczWwxzzV0D3rmITGXtG0UAt/ug= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/examples/hotrod/README.md b/examples/hotrod/README.md index b0faee983713..9fe042d03f0f 100644 --- a/examples/hotrod/README.md +++ b/examples/hotrod/README.md @@ -47,8 +47,8 @@ Jaeger UI can be accessed at http://localhost:16686. ```bash git clone git@github.com:jaegertracing/jaeger.git jaeger -cd jaeger/examples/hotrod -go run ./main.go all +cd jaeger +go run ./examples/hotrod/main.go all ``` ### Run HotROD from docker @@ -73,3 +73,14 @@ The app exposes metrics in either Go's `expvar` format (by default) or in Promet [hotrod-tutorial]: https://medium.com/@YuriShkuro/take-opentracing-for-a-hotrod-ride-f6e3141f7941 [hotrod-openshift]: https://blog.openshift.com/openshift-commons-briefing-82-distributed-tracing-with-jaeger-prometheus-on-kubernetes/ + +## Linking to traces + +The HotROD UI can generate links to the Jaeger UI to find traces corresponding +to each executed request. By default it uses the standard Jaeger UI address +http://localhost:16686, but if your Jaeger UI is running at a different address, +it can be customized via `-j
` flag passed to HotROD, e.g. + +``` +go run ./examples/hotrod/main.go all -j http://jaeger-ui:16686 +``` diff --git a/examples/hotrod/cmd/frontend.go b/examples/hotrod/cmd/frontend.go index 6ff6551af6d6..273b15617e91 100644 --- a/examples/hotrod/cmd/frontend.go +++ b/examples/hotrod/cmd/frontend.go @@ -39,6 +39,7 @@ var frontendCmd = &cobra.Command{ options.CustomerHostPort = net.JoinHostPort("0.0.0.0", strconv.Itoa(customerPort)) options.RouteHostPort = net.JoinHostPort("0.0.0.0", strconv.Itoa(routePort)) options.Basepath = basepath + options.JaegerUI = jaegerUI zapLogger := logger.With(zap.String("service", "frontend")) logger := log.NewFactory(zapLogger) diff --git a/examples/hotrod/cmd/root.go b/examples/hotrod/cmd/root.go index 80342be2ea75..68b363ade08a 100644 --- a/examples/hotrod/cmd/root.go +++ b/examples/hotrod/cmd/root.go @@ -45,6 +45,7 @@ var ( routePort int basepath string + jaegerUI string ) // RootCmd represents the base command when called without any subcommands @@ -77,6 +78,7 @@ func init() { // Flag for serving frontend at custom basepath url RootCmd.PersistentFlags().StringVarP(&basepath, "basepath", "b", "", `Basepath for frontend service(default "/")`) + RootCmd.PersistentFlags().StringVarP(&jaegerUI, "jaeger-ui", "j", "http://localhost:16686", "Address of Jaeger UI to create [find trace] links") rand.Seed(int64(time.Now().Nanosecond())) logger, _ = zap.NewDevelopment( diff --git a/examples/hotrod/docker-compose.yml b/examples/hotrod/docker-compose.yml index 427047245759..c2634ce56b4f 100644 --- a/examples/hotrod/docker-compose.yml +++ b/examples/hotrod/docker-compose.yml @@ -14,6 +14,8 @@ services: command: ["all"] environment: - JAEGER_AGENT_HOST=jaeger + # Note: if your application is using Node.js Jaeger Client, you need port 6832, + # unless issue https://github.com/jaegertracing/jaeger/issues/1596 is resolved. - JAEGER_AGENT_PORT=6831 networks: - jaeger-example diff --git a/examples/hotrod/services/frontend/gen_assets.go b/examples/hotrod/services/frontend/gen_assets.go index f2899b5219bd..b924f0754ea9 100644 --- a/examples/hotrod/services/frontend/gen_assets.go +++ b/examples/hotrod/services/frontend/gen_assets.go @@ -212,34 +212,37 @@ var _escData = map[string]*_escFile{ "/index.html": { name: "index.html", local: "examples/hotrod/services/frontend/web_assets/index.html", - size: 3530, - modtime: 1597936165, + size: 4058, + modtime: 1601939528, compressed: ` -H4sIAAAAAAAC/9RXX1PbSBJ/96fom82d5DOSMIZAjOUtDmcJ2cuSM5Ct3FYeRqO2NEaaUWZGtlmK7341 -+mNkSKru3i5+gJnunu7+9V97kpo8m/YAJjkaCiylSqMJyeX1lXdycvTGG5InrqA5hmTFcV1IZQgwKQwK -E5I1j00axrjiDL3qsgdccMNp5mlGMwyH/v4e5HTD8zLvkkqNqrrTKMNwvzaWIo3tAWBiuMlw+k6a+dUM -PJjzGDVcCZhhTkU8CWp+LauZ4oUBrVhI/GD5tUR17438oT/0cy78pSbTSVALNS8yLu5AYRYSbe4z1Cmi -IZAqXIQkNabQ4yDI6YbFwo+kNNooWtgLk3mwJQQjf+QfB0zrJ1plkGlNgAuDieLmPiQ6paOTQ+8fnz5z -fn35C/46jC/y9/Ozu3tWvjt7N09GB1f5LVuvj6UYzT/HyeEnOviYX9/oP4NfX5+sovjtMj0sCTAltZaK -J1yEhAop7nNZavKNMPyvIJbPMSy/CeGGHV3+i0f7B8dfV/fL6w+Ld8urD/Sfd4vy90+bf29uP4rz92fH -2UF+/vtvl8XFm/zifHayvvjtkn2cHd9s6PchPCWoAWPzMu35ZcljeICcqoQLz8hiDMOjYnMKjz0/lUbJ -2ItKY6SAByhoHHORjOFg30qwUmmpxlBIC0Sd7irZ/5aScSpXqODh5dsFzwyqMUSKJ6kRqLV7cvTXvlXx -U6Mik8l3PP3J8OI7rAps0KC1PRC0TTCJZHzfpDbmK2AZ1ToktvcoF6iatO9yq3DRDJWp/3pcLKSNbsxX -W3mGFlN7tX03tJ0Gc//Kn/mTIB12eYfTCebTFw2I+XQSpIcdyY4bSq7JE+clhMzLY28E9qBz7/Uz2boA -CipeUO2nURIZAZERFcDqEGWS3cFOOsk3FcTUUI+V2sgcVUiGByMynVOWYuZo+CWTimYwQ80ToSeBdeMZ -km4s/9/Bjd4ckOmNkjmcp5LJjBqO6odHdTwakul7WlCBGm2uNCrz4yfr6PUxmZ7l9E8uEjiXiwUizCXV -BtV/A+751eLkcUgML8j0POPsDqSA1ly11IFGcoVgJEgVowIKjCr/e4qe5hxpsWdI4+fjJejOly1rEtTz -rLfdVNNeb1EKZrgUsJAqp2ZWKmqvbtwc+vDQA1hRBTGE0FIhAHe4X33g7zCs/73e7582sqXgRkMITs6F -Y4kKTakEfKAm9ZUsRezGfRjUcqe9x17PvmIZR2Fuby9nEHZF6yMVsczdfmPP2rJvMqrNHL+WqE31bP+0 -13vlkmprkb5vv2O55LMsFawxaiw4Gng8thtOSZFMCQy6pgdA7DqoWf1G3W4p9X1mk+m2wXNxZepA7bgz -GLTxUB0Pd0151vrOq/bNQqFOz6mCEF65r1zSWXKk7xcKCxSx63SbqXriMapItTFmXBfUsNQWc11Xvv+H -wq9jcAZbjwbOl2aV2DJx+j5LeRYrFG7/j/0v24xuizYEXBnfUJWg8W37aDR+y22l7QZFZSvgoalJZ0kx -QeVFNElogs4YHI1acylC53n8nb02YBVv62oP4NFaYFJomaGfycRtLG39jHAhFUIIM2rQF3Lt2hQCBAHc -arTfKhQKA7fzS6AaIqqxoCa11Q90STetMd2os8yPChd8AyGsuYjl2s8kq5rAt0zbw9b2jmDn8pcQSEDg -5y5tDI5TOfXKtzbdDmsAThA3aft5O5eqCLUJGIDzNyGFxoq80xt7TbiboIzbw15FzdGkMh6Dc/H2xqlJ -umQMtR7DtpJtRvfA4MZcG2pK3d9m0IaDLkxVAt3gNpO4kxOrY8uoRkc7NsIXY4Ya6r+9OduKt1Vfd67T -/PyYRFMLtpKeKb6qwzAJoilQpfjKVjgXUMm0tgbgQFPt3SKq6yujBgW7r3luBcurS8eOJSfXX5zGpUcb -qcf+ae+xLqTO1+RJUP+E+08AAAD//1QoAETKDQAA +H4sIAAAAAAAC/9RXbVPjOPJ/n0/Rf23AyT/EzgMMTIgzxZJZhtmdZS7AbM1NUYcsd2yBLWUkOYFN5btf +yXaCA0zd3au7zYtE6m5196+fFA1jkyajGsAwRUOBxVRpND45v7xoHx0dvG13yRNX0BR9Mue4mEllCDAp +DArjkwUPTeyHOOcM2/lmD7jghtOkrRlN0O+6nT1I6QNPs7RKyjSqfE+DBP1OYSxGGtoFwNBwk+DogzST +izG0YcJD1HAhYIwpFeHQK/iFrGaKzwxoxXzienffM1SP7b7bdbtuyoV7p8lo6BVC5YmEi3tQmPhEm8cE +dYxoCMQKpz6JjZnpgeel9IGFwg2kNNooOrMbJlNvQ/D6bt899JjWT7TcINOaABcGI8XNo090TPtH++2f +v3zl/PL8F/y1G56lHycn948s+3DyYRL1exfpNVssDqXoT76G0f4X2vqcXl7pP71f3xzNg/D9XbyfEWBK +ai0Vj7jwCRVSPKYy0+SVMPynIO6eY7h7FcIVOzj/Gw86vcPv88e7y0/TD3cXn+hv99Psjy8Pf3+4/ixO +P54cJr309I/fz2dnb9Oz0/HR4uz3c/Z5fHj1QH8M4SlBJRibl1HNzTIewhJSqiIu2kbOBtA9mD0cw6rm +xtIoGbaDzBgpYAkzGoZcRAPodawEy5SWagAzaYGo420lndeUDGI5RwXLl2enPDGoBhAoHsVGoNaNo4Od +plXxU6kikdEPPP3J8NkPWDlYr0Rre8BbN8EwkOFjmdqQz4ElVGuf2N6jXKAq077NzcNFE1Sm+G5zMZU2 +uiGfb+QZWkzrre27ru00mLgX7tgdenG3ytsfDTEdvWhATEdDL96vSFbcUHJBnjgvISTtNGz3wS502n7z +TLYogBkVL6j2UyoJjIDAiBxgvggSye5hK53kVQUhNbTNMm1kison3V6fjCaUxZg4Gn5JpKIJjFHzSOih +Z914hqQay/91cP23PTK6UjKF01gymVDDUf3lUR32u2T0kc6oQI02VxqV+esn6+DNIRmdpPRPLiI4ldMp +Ikwk1QbVvwPu+dbi5KFPDJ+R0WnC2T1IAWtz+aUONJBzBCNBqhAVUGBUuT9S9DTnyBp7gjR8Pl686nzZ +sIZeMc9qm5tqVKtNM8EMlwKmUqXUjDNF7bYRlosmLGsAc6ogBB/WVPCg0e3kH/h/6BY/bzrN41I2E9xo +8MFJuXAsUaHJlIBP1MSukpkIG2ETWoXccW1Vq9lTLOEozPX1+Rj8qmixpCKUaaNZ2rO27JmEajPB7xlq +kx/rHNdq9QbJby3SdO1/rMbtV5kpWGBQWnA08HBgbzglRTSqL58Mr+xNkFNvm6Wm7SpquszmsbGOWwPn +pojRliet1joUquJcBWALSJtAa/vU+sxUoY5PqQIf6o16g1TuN9J0ZwpnKMKGU+2j/EibUUXyy2LM9Ywa +Fts6LkrKdb8p/D4Ap7XxqOXclLeIrRCn6bKYJ6FC0Wh+69xskrmpVx9wblxDVYTGtZ2j0bhr7lraXp6o +bPKXZTk6dxQjVO2ARhGN0BmAo1FrLoXvQGs7KM7eOmA5b+NqDWBlLTAptEzQTWTUKC1t/AxwKhWCD2Nq +0BVy0bApBPA8uNZo/1AoFAauJ+dANQRU44ya2BY+0Dv6sDamS3WW+VnhlD+ADwsuQrlwE8ny+nct07av +tb0lWNn8nw/EI/CuShuA4+RO5YGVYsojG6kcW921bjQq0i1wvELonZBCYx6TrW7YK6OcoollOADn7P2V +s5eTdMYYaj2ATanalO2BwQdzaajJdHOTIusNnZo8x9XolVO2EnSro8ooAFhqQVxZ46si8K8DCsvSfLcZ +u3kVrIusBc7uvwBbJn6wXuz9F0OQT8b1VPRfTFFqqPv+6mRL3CjK8Df7/vHzaig4fAqNMp67u2Vkv5Wt +49w8+VnoKOh2pDwTPN4Sy02BD7f1ZSGw8jRSxeJ3CU+58Xud3UTK+4Cye78b72pU9h3pT1X+vAx3DY20 +v3P4806vFyo+R7XT6+30T3Z6vfoyxzbOqStLPhzfPhmvYryFb0NaPu3qy5yzIlCMEZ/8I0iouCejKRdh +cWzo0dHNRteq/F3PxHKkl+/SYTDa9mToBSOgSvG5HX1cQH25Ts8KiglYX27GymoPEmpQsEdLztPfLsbI +KtU3UHprcaxum88KvCzyyrNp6BVP+n8GAAD//wz8Z3XaDwAA `, }, @@ -247,7 +250,7 @@ qcf+ae+xLqTO1+RJUP+E+08AAAD//1QoAETKDQAA name: "jquery-3.1.1.min.js", local: "examples/hotrod/services/frontend/web_assets/jquery-3.1.1.min.js", size: 86709, - modtime: 1597936165, + modtime: 1598995573, compressed: ` H4sIAAAAAAAC/8y9fZebONIo/v/9FG02D4PaMm1nZvbewa1wMslkN7vztpPMzO5iskeAwLgxuAGnO2PY z/47KkkgME5mn+fec34nJ21A71KpVFWql5vr2dXub0dWfrh6/7m9sldXzZUVIvXtVXHMI1qnRX7VXO3u diff --git a/examples/hotrod/services/frontend/server.go b/examples/hotrod/services/frontend/server.go index 4c2ce661279b..e6e22dc31c17 100644 --- a/examples/hotrod/services/frontend/server.go +++ b/examples/hotrod/services/frontend/server.go @@ -36,6 +36,7 @@ type Server struct { bestETA *bestETA assetFS http.FileSystem basepath string + jaegerUI string } // ConfigOptions used to make sure service clients @@ -46,6 +47,7 @@ type ConfigOptions struct { CustomerHostPort string RouteHostPort string Basepath string + JaegerUI string } // NewServer creates a new frontend.Server @@ -58,6 +60,7 @@ func NewServer(options ConfigOptions, tracer opentracing.Tracer, logger log.Fact bestETA: newBestETA(tracer, logger, options), assetFS: assetFS, basepath: options.Basepath, + jaegerUI: options.JaegerUI, } } @@ -73,9 +76,17 @@ func (s *Server) createServeMux() http.Handler { p := path.Join("/", s.basepath) mux.Handle(p, http.StripPrefix(p, http.FileServer(s.assetFS))) mux.Handle(path.Join(p, "/dispatch"), http.HandlerFunc(s.dispatch)) + mux.Handle(path.Join(p, "/config"), http.HandlerFunc(s.config)) return mux } +func (s *Server) config(w http.ResponseWriter, r *http.Request) { + config := map[string]string{ + "jaeger": s.jaegerUI, + } + s.writeResponse(config, w, r) +} + func (s *Server) dispatch(w http.ResponseWriter, r *http.Request) { ctx := r.Context() s.logger.For(ctx).Info("HTTP request received", zap.String("method", r.Method), zap.Stringer("url", r.URL)) @@ -97,9 +108,13 @@ func (s *Server) dispatch(w http.ResponseWriter, r *http.Request) { return } + s.writeResponse(response, w, r) +} + +func (s *Server) writeResponse(response interface{}, w http.ResponseWriter, r *http.Request) { data, err := json.Marshal(response) if httperr.HandleError(w, err, http.StatusInternalServerError) { - s.logger.For(ctx).Error("cannot marshal response", zap.Error(err)) + s.logger.For(r.Context()).Error("cannot marshal response", zap.Error(err)) return } diff --git a/examples/hotrod/services/frontend/web_assets/index.html b/examples/hotrod/services/frontend/web_assets/index.html index 84ff4d8dc92c..de55d32e78eb 100644 --- a/examples/hotrod/services/frontend/web_assets/index.html +++ b/examples/hotrod/services/frontend/web_assets/index.html @@ -61,7 +61,7 @@

Rides On Demand

var clientUUID = Math.round(Math.random() * 10000); var lastRequestID = 0; -$(".uuid").html("Your web client's id: " + clientUUID + ""); +$(".uuid").html(`Your web client's id: ${clientUUID}`); $(".hotrod-button").click(function(evt) { lastRequestID++; @@ -78,6 +78,16 @@

Rides On Demand

var pathPrefix = window.location.pathname; pathPrefix = pathPrefix != "/" ? pathPrefix : ''; + var config = {}; + $.ajax(pathPrefix + '/config?nonse=' + Math.random(), { + method: 'GET', + success: function(data, textStatus) { + var after = Date.now(); + console.log(data); + config = data; + }, + }); + $.ajax(pathPrefix + '/dispatch?customer=' + customer + '&nonse=' + Math.random(), { headers: headers, method: 'GET', @@ -85,7 +95,13 @@

Rides On Demand

var after = Date.now(); console.log(data); var duration = formatDuration(data.ETA); - freshCar.html('HotROD ' + data.Driver + ' arriving in ' + duration + ' [req: ' + requestID + ', latency: ' + (after-before) + 'ms]'); + var traceLink = ''; + if (config && config['jaeger']) { + var jaeger = config['jaeger']; + var trace = `${jaeger}/search?limit=20&lookback=1h&service=frontend&tags=%7B%22driver%22%3A%22${data.Driver}%22%7D`; + traceLink = ` [find trace]`; + } + freshCar.html(`HotROD ${data.Driver} arriving in ${duration} [req: ${requestID}, latency: ${after-before}ms] ${traceLink}`); }, }); }); diff --git a/go.mod b/go.mod index ebbcfa555dd6..df7c029612e6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/DataDog/zstd v1.4.4 // indirect + github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect github.com/Shopify/sarama v1.22.2-0.20190604114437-cd910a683f9f github.com/apache/thrift v0.13.0 github.com/bsm/sarama-cluster v2.1.13+incompatible @@ -66,7 +67,7 @@ require ( github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.5.1 github.com/uber/jaeger-client-go v2.23.1+incompatible - github.com/uber/jaeger-lib v2.2.0+incompatible + github.com/uber/jaeger-lib v2.4.0+incompatible github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad go.mongodb.org/mongo-driver v1.3.2 // indirect diff --git a/go.sum b/go.sum index d3fb57363453..62214b83bda9 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg= +github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -560,8 +562,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.23.1+incompatible h1:uArBYHQR0HqLFFAypI7RsWTzPSj/bDpmZZuQjMLSg1A= github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= -github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= +github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= diff --git a/jaeger-ui b/jaeger-ui index 8279218e921b..fd2160653755 160000 --- a/jaeger-ui +++ b/jaeger-ui @@ -1 +1 @@ -Subproject commit 8279218e921b25b6700698167f75a07105e8d5e8 +Subproject commit fd2160653755f2d2ffa3466df53b080eda22badf diff --git a/plugin/sampling/strategystore/static/options.go b/plugin/sampling/strategystore/static/options.go index 0ef18fbf2ea3..b7222b3a5c20 100644 --- a/plugin/sampling/strategystore/static/options.go +++ b/plugin/sampling/strategystore/static/options.go @@ -22,7 +22,7 @@ import ( ) const ( - // SamplingStrategiesFile contains the name of CLI opions for config file. + // SamplingStrategiesFile contains the name of CLI option for config file. SamplingStrategiesFile = "sampling.strategies-file" samplingStrategiesReloadInterval = "sampling.strategies-reload-interval" ) diff --git a/plugin/sampling/strategystore/static/strategy_store.go b/plugin/sampling/strategystore/static/strategy_store.go index 561e7cb91b42..fb78541600ec 100644 --- a/plugin/sampling/strategystore/static/strategy_store.go +++ b/plugin/sampling/strategystore/static/strategy_store.go @@ -21,7 +21,8 @@ import ( "encoding/json" "fmt" "io/ioutil" - "path/filepath" + "net/http" + "net/url" "sync/atomic" "time" @@ -45,6 +46,8 @@ type storedStrategies struct { serviceStrategies map[string]*sampling.SamplingStrategyResponse } +type strategyLoader func() ([]byte, error) + // NewStrategyStore creates a strategy store that holds static sampling strategies. func NewStrategyStore(options Options, logger *zap.Logger) (ss.StrategyStore, error) { ctx, cancelFunc := context.WithCancel(context.Background()) @@ -55,14 +58,15 @@ func NewStrategyStore(options Options, logger *zap.Logger) (ss.StrategyStore, er } h.storedStrategies.Store(defaultStrategies()) - strategies, err := loadStrategies(options.StrategiesFile) + loader := samplingStrategyLoader(options.StrategiesFile) + strategies, err := loadStrategies(loader) if err != nil { return nil, err } h.parseStrategies(strategies) if options.ReloadInterval > 0 { - go h.autoUpdateStrategies(options.ReloadInterval, options.StrategiesFile) + go h.autoUpdateStrategies(options.ReloadInterval, loader) } return h, nil } @@ -83,35 +87,86 @@ func (h *strategyStore) Close() { h.cancelFunc() } -func (h *strategyStore) autoUpdateStrategies(interval time.Duration, filePath string) { - lastValue := "" +func downloadSamplingStrategies(url string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to download sampling strategies: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusServiceUnavailable { + return []byte("null"), nil + } + return nil, fmt.Errorf( + "receiving %s while downloading strategies file", + resp.Status, + ) + } + + buf := new(bytes.Buffer) + if _, err = buf.ReadFrom(resp.Body); err != nil { + return nil, fmt.Errorf("failed to read sampling strategies from downloaded JSON: %w", err) + } + return buf.Bytes(), nil +} + +func isURL(str string) bool { + u, err := url.Parse(str) + return err == nil && u.Scheme != "" && u.Host != "" +} + +func samplingStrategyLoader(strategiesFile string) strategyLoader { + return func() ([]byte, error) { + if strategiesFile == "" { + // Using "null" so that it un-marshals to nil pointer. + return []byte("null"), nil + } + + if ok := isURL(strategiesFile); ok { + currBytes, err := downloadSamplingStrategies(strategiesFile) + if err != nil { + return nil, err + } + return currBytes, nil + } + + currBytes, err := ioutil.ReadFile(strategiesFile) + if err != nil { + return nil, fmt.Errorf("failed to open strategies file: %w", err) + } + return currBytes, nil + } +} + +func (h *strategyStore) autoUpdateStrategies(interval time.Duration, loader strategyLoader) { + lastValue := "null" ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ticker.C: - lastValue = h.reloadSamplingStrategyFile(filePath, lastValue) + lastValue = h.reloadSamplingStrategy(loader, lastValue) case <-h.ctx.Done(): return } } } -func (h *strategyStore) reloadSamplingStrategyFile(filePath string, lastValue string) string { - currBytes, err := ioutil.ReadFile(filepath.Clean(filePath)) +func (h *strategyStore) reloadSamplingStrategy(loader strategyLoader, lastValue string) string { + newValue, err := loader() if err != nil { - h.logger.Error("failed to load sampling strategies", zap.String("file", filePath), zap.Error(err)) + h.logger.Error("failed to re-load sampling strategies", zap.Error(err)) return lastValue } - newValue := string(currBytes) - if lastValue == newValue { + if lastValue == string(newValue) { return lastValue } - if err = h.updateSamplingStrategy(currBytes); err != nil { - h.logger.Error("failed to update sampling strategies from file", zap.Error(err)) + if err := h.updateSamplingStrategy(newValue); err != nil { + h.logger.Error("failed to update sampling strategies", zap.Error(err)) return lastValue } - return newValue + return string(newValue) } func (h *strategyStore) updateSamplingStrategy(bytes []byte) error { @@ -125,24 +180,22 @@ func (h *strategyStore) updateSamplingStrategy(bytes []byte) error { } // TODO good candidate for a global util function -func loadStrategies(strategiesFile string) (*strategies, error) { - if strategiesFile == "" { - return nil, nil - } - data, err := ioutil.ReadFile(strategiesFile) /* nolint #nosec , this comes from an admin, not user */ +func loadStrategies(loader strategyLoader) (*strategies, error) { + strategyBytes, err := loader() if err != nil { - return nil, fmt.Errorf("failed to open strategies file: %w", err) + return nil, err } - var strategies strategies - if err := json.Unmarshal(data, &strategies); err != nil { + + var strategies *strategies + if err := json.Unmarshal(strategyBytes, &strategies); err != nil { return nil, fmt.Errorf("failed to unmarshal strategies: %w", err) } - return &strategies, nil + return strategies, nil } func (h *strategyStore) parseStrategies(strategies *strategies) { if strategies == nil { - h.logger.Info("No sampling strategies provided, using defaults") + h.logger.Info("No sampling strategies provided or URL is unavailable, using defaults") return } newStore := defaultStrategies() diff --git a/plugin/sampling/strategystore/static/strategy_store_test.go b/plugin/sampling/strategystore/static/strategy_store_test.go index 29a96c474c26..5b6e79ffd7a4 100644 --- a/plugin/sampling/strategystore/static/strategy_store_test.go +++ b/plugin/sampling/strategystore/static/strategy_store_test.go @@ -17,6 +17,8 @@ package static import ( "context" "io/ioutil" + "net/http" + "net/http/httptest" "os" "strings" "testing" @@ -31,6 +33,37 @@ import ( "github.com/jaegertracing/jaeger/thrift-gen/sampling" ) +// Returns strategies in JSON format. Used for testing +// URL option for sampling strategies. +func mockStrategyServer() *httptest.Server { + f := func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/bad-content": + w.Write([]byte("bad-content")) + return + + case "/bad-status": + w.WriteHeader(404) + return + + case "/service-unavailable": + w.WriteHeader(503) + return + + default: + data, err := ioutil.ReadFile("fixtures/strategies.json") + if err != nil { + w.WriteHeader(500) + return + } + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json") + w.Write(data) + } + } + return httptest.NewServer(http.HandlerFunc(f)) +} + func TestStrategyStore(t *testing.T) { _, err := NewStrategyStore(Options{StrategiesFile: "fileNotFound.json"}, zap.NewNop()) assert.EqualError(t, err, "failed to open strategies file: open fileNotFound.json: no such file or directory") @@ -43,7 +76,7 @@ func TestStrategyStore(t *testing.T) { logger, buf := testutils.NewLogger() store, err := NewStrategyStore(Options{}, logger) require.NoError(t, err) - assert.Contains(t, buf.String(), "No sampling strategies provided, using defaults") + assert.Contains(t, buf.String(), "No sampling strategies provided or URL is unavailable, using defaults") s, err := store.GetSamplingStrategy(context.Background(), "foo") require.NoError(t, err) assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.001), *s) @@ -62,6 +95,26 @@ func TestStrategyStore(t *testing.T) { s, err = store.GetSamplingStrategy(context.Background(), "default") require.NoError(t, err) assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.5), *s) + + // Test default strategy when URL is temporarily unavailable. + mockServer := mockStrategyServer() + store, err = NewStrategyStore(Options{StrategiesFile: mockServer.URL+"/service-unavailable"}, logger) + assert.Contains(t, buf.String(), "No sampling strategies provided or URL is unavailable, using defaults") + s, err = store.GetSamplingStrategy(context.Background(), "foo") + require.NoError(t, err) + assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.001), *s) + + // Test downloading strategies from a URL. + store, err = NewStrategyStore(Options{StrategiesFile: mockServer.URL}, logger) + require.NoError(t, err) + + s, err = store.GetSamplingStrategy(context.Background(), "foo") + require.NoError(t, err) + assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.8), *s) + + s, err = store.GetSamplingStrategy(context.Background(), "bar") + require.NoError(t, err) + assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_RATE_LIMITING, 5), *s) } func TestPerOperationSamplingStrategies(t *testing.T) { @@ -276,7 +329,7 @@ func TestAutoUpdateStrategy(t *testing.T) { assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.8), *s) // verify that reloading in no-op - value := store.reloadSamplingStrategyFile(dstFile, string(srcBytes)) + value := store.reloadSamplingStrategy(samplingStrategyLoader(dstFile), string(srcBytes)) assert.Equal(t, string(srcBytes), value) // update file with new probability of 0.9 @@ -293,6 +346,49 @@ func TestAutoUpdateStrategy(t *testing.T) { time.Sleep(1 * time.Millisecond) } assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.9), *s) + + // Test auto update strategy with URL option. + mockServer := mockStrategyServer() + ss, err = NewStrategyStore(Options{ + StrategiesFile: mockServer.URL, + ReloadInterval: 10 * time.Millisecond, + }, zap.NewNop()) + require.NoError(t, err) + store = ss.(*strategyStore) + defer store.Close() + + // copy existing fixture content to restore it later. + srcBytes, err = ioutil.ReadFile(srcFile) + require.NoError(t, err) + originalBytes := srcBytes + + // confirm baseline value + s, err = store.GetSamplingStrategy(context.Background(), "foo") + require.NoError(t, err) + assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.8), *s) + + // verify that reloading in no-op + value = store.reloadSamplingStrategy(samplingStrategyLoader(mockServer.URL), string(srcBytes)) + assert.Equal(t, string(srcBytes), value) + + // update original strategies file with new probability of 0.9 + newStr = strings.Replace(string(srcBytes), "0.8", "0.9", 1) + require.NoError(t, ioutil.WriteFile(srcFile, []byte(newStr), 0644)) + defer func() { + // replace original strategies file with old content. + require.NoError(t, ioutil.WriteFile(srcFile, originalBytes, 0644), "failed to restore original file content") + }() + + // wait for reload timer + for i := 0; i < 1000; i++ { // wait up to 1sec + s, err = store.GetSamplingStrategy(context.Background(), "foo") + require.NoError(t, err) + if s.ProbabilisticSampling != nil && s.ProbabilisticSampling.SamplingRate == 0.9 { + break + } + time.Sleep(1 * time.Millisecond) + } + assert.EqualValues(t, makeResponse(sampling.SamplingStrategyType_PROBABILISTIC, 0.9), *s) } func TestAutoUpdateStrategyErrors(t *testing.T) { @@ -314,13 +410,25 @@ func TestAutoUpdateStrategyErrors(t *testing.T) { defer store.Close() // check invalid file path or read failure - assert.Equal(t, "blah", store.reloadSamplingStrategyFile(tempFile.Name()+"bad-path", "blah")) - assert.Len(t, logs.FilterMessage("failed to load sampling strategies").All(), 1) + assert.Equal(t, "blah", store.reloadSamplingStrategy(samplingStrategyLoader(tempFile.Name()+"bad-path"), "blah")) + assert.Len(t, logs.FilterMessage("failed to re-load sampling strategies").All(), 1) // check bad file content require.NoError(t, ioutil.WriteFile(tempFile.Name(), []byte("bad value"), 0644)) - assert.Equal(t, "blah", store.reloadSamplingStrategyFile(tempFile.Name(), "blah")) - assert.Len(t, logs.FilterMessage("failed to update sampling strategies from file").All(), 1) + assert.Equal(t, "blah", store.reloadSamplingStrategy(samplingStrategyLoader(tempFile.Name()), "blah")) + assert.Len(t, logs.FilterMessage("failed to update sampling strategies").All(), 1) + + // check invalid url + assert.Equal(t, "duh", store.reloadSamplingStrategy(samplingStrategyLoader("bad-url"), "duh")) + assert.Len(t, logs.FilterMessage("failed to re-load sampling strategies").All(), 2) + + // check status code other than 200 + assert.Equal(t, "duh", store.reloadSamplingStrategy(samplingStrategyLoader(mockStrategyServer().URL+"/bad-status"), "duh")) + assert.Len(t, logs.FilterMessage("failed to re-load sampling strategies").All(), 3) + + // check bad content from url + assert.Equal(t, "duh", store.reloadSamplingStrategy(samplingStrategyLoader(mockStrategyServer().URL+"/bad-content"), "duh")) + assert.Len(t, logs.FilterMessage("failed to update sampling strategies").All(), 2) } func TestServiceNoPerOperationStrategies(t *testing.T) { @@ -337,3 +445,21 @@ func TestServiceNoPerOperationStrategies(t *testing.T) { expected := makeResponse(sampling.SamplingStrategyType_RATE_LIMITING, 3) assert.Equal(t, *expected.RateLimitingSampling, *s.RateLimitingSampling) } + +func TestSamplingStrategyLoader(t *testing.T) { + // invalid file path + loader := samplingStrategyLoader("not-exists") + _, err := loader() + assert.Contains(t, err.Error(), "failed to open strategies file") + + // status code other than 200 + mockServer := mockStrategyServer() + loader = samplingStrategyLoader(mockServer.URL + "/bad-status") + _, err = loader() + assert.Contains(t, err.Error(), "receiving 404 Not Found while downloading strategies file") + + // should download content from URL + loader = samplingStrategyLoader(mockServer.URL + "/bad-content") + content, err := loader() + assert.Equal(t, "bad-content", string(content)) +} diff --git a/plugin/storage/badger/spanstore/reader.go b/plugin/storage/badger/spanstore/reader.go index 134da6e44c27..4571497bf9a4 100644 --- a/plugin/storage/badger/spanstore/reader.go +++ b/plugin/storage/badger/spanstore/reader.go @@ -264,23 +264,28 @@ func setQueryDefaults(query *spanstore.TraceQueryParameters) { func serviceQueries(query *spanstore.TraceQueryParameters, indexSeeks [][]byte) [][]byte { if query.ServiceName != "" { indexSearchKey := make([]byte, 0, 64) // 64 is a magic guess + tagQueryUsed := false + for k, v := range query.Tags { + tagSearch := []byte(query.ServiceName + k + v) + tagSearchKey := make([]byte, 0, len(tagSearch)+1) + tagSearchKey = append(tagSearchKey, tagIndexKey) + tagSearchKey = append(tagSearchKey, tagSearch...) + indexSeeks = append(indexSeeks, tagSearchKey) + tagQueryUsed = true + } + if query.OperationName != "" { indexSearchKey = append(indexSearchKey, operationNameIndexKey) indexSearchKey = append(indexSearchKey, []byte(query.ServiceName+query.OperationName)...) } else { - indexSearchKey = append(indexSearchKey, serviceNameIndexKey) - indexSearchKey = append(indexSearchKey, []byte(query.ServiceName)...) + if !tagQueryUsed { // Tag query already reduces the search set with a serviceName + indexSearchKey = append(indexSearchKey, serviceNameIndexKey) + indexSearchKey = append(indexSearchKey, []byte(query.ServiceName)...) + } } - indexSeeks = append(indexSeeks, indexSearchKey) - if len(query.Tags) > 0 { - for k, v := range query.Tags { - tagSearch := []byte(query.ServiceName + k + v) - tagSearchKey := make([]byte, 0, len(tagSearch)+1) - tagSearchKey = append(tagSearchKey, tagIndexKey) - tagSearchKey = append(tagSearchKey, tagSearch...) - indexSeeks = append(indexSeeks, tagSearchKey) - } + if len(indexSearchKey) > 0 { + indexSeeks = append(indexSeeks, indexSearchKey) } } return indexSeeks diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 06da56a3d8f5..c575befc8a4a 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -179,9 +179,9 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { flagSet.Int( nsConfig.namespace+suffixMaxNumSpans, nsConfig.MaxDocCount, - "(deprecated, will be removed in release v1.21.0. Please use es.max-doc-count). "+ + "(deprecated, will be removed in release v1.21.0. Please use "+nsConfig.namespace+".max-doc-count). "+ "The maximum number of spans to fetch at a time per query in Elasticsearch. "+ - "The lesser of es.max-num-spans and es.max-doc-count will be used if both are set.") + "The lesser of "+nsConfig.namespace+".max-num-spans and "+nsConfig.namespace+".max-doc-count will be used if both are set.") flagSet.Int64( nsConfig.namespace+suffixNumShards, nsConfig.NumShards, diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index 65ac35e405a9..a7148a82e680 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -96,6 +96,33 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "test,tags", aux.Tags.Include) } +func TestMaxNumSpansUsage(t *testing.T) { + testCases := []struct { + namespace string + wantUsage string + }{ + { + namespace: "es", + wantUsage: "(deprecated, will be removed in release v1.21.0. Please use es.max-doc-count). " + + "The maximum number of spans to fetch at a time per query in Elasticsearch. " + + "The lesser of es.max-num-spans and es.max-doc-count will be used if both are set.", + }, + { + namespace: "es-archive", + wantUsage: "(deprecated, will be removed in release v1.21.0. Please use es-archive.max-doc-count). " + + "The maximum number of spans to fetch at a time per query in Elasticsearch. " + + "The lesser of es-archive.max-num-spans and es-archive.max-doc-count will be used if both are set.", + }, + } + for _, tc := range testCases { + t.Run(tc.namespace, func(t *testing.T) { + opts := NewOptions(tc.namespace) + _, command := config.Viperize(opts.AddFlags) + assert.Equal(t, tc.wantUsage, command.Flag(tc.namespace+".max-num-spans").Usage) + }) + } +} + func TestMaxDocCount(t *testing.T) { testCases := []struct { name string diff --git a/plugin/storage/factory.go b/plugin/storage/factory.go index 5a7062001e84..c56c8ef50ac9 100644 --- a/plugin/storage/factory.go +++ b/plugin/storage/factory.go @@ -46,6 +46,7 @@ const ( badgerStorageType = "badger" downsamplingRatio = "downsampling.ratio" downsamplingHashSalt = "downsampling.hashsalt" + spanStorageType = "span-storage-type" // defaultDownsamplingRatio is the default downsampling ratio. defaultDownsamplingRatio = 1.0 @@ -111,6 +112,8 @@ func (f *Factory) Initialize(metricsFactory metrics.Factory, logger *zap.Logger) return err } } + f.publishOpts() + return nil } @@ -249,3 +252,11 @@ func (f *Factory) Close() error { } return multierror.Wrap(errs) } + +func (f *Factory) publishOpts() { + internalFactory := f.metricsFactory.Namespace(metrics.NSOptions{Name: "internal"}) + internalFactory.Gauge(metrics.Options{Name: downsamplingRatio}). + Update(int64(f.FactoryConfig.DownsamplingRatio)) + internalFactory.Gauge(metrics.Options{Name: spanStorageType + "-" + f.SpanReaderType}). + Update(1) +} diff --git a/plugin/storage/factory_test.go b/plugin/storage/factory_test.go index a4effa5f9d41..6728ac88938b 100644 --- a/plugin/storage/factory_test.go +++ b/plugin/storage/factory_test.go @@ -23,11 +23,14 @@ import ( "reflect" "strings" "testing" + "time" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber/jaeger-lib/metrics" + "github.com/uber/jaeger-lib/metrics/fork" + "github.com/uber/jaeger-lib/metrics/metricstest" "go.uber.org/zap" "github.com/jaegertracing/jaeger/pkg/config" @@ -350,6 +353,28 @@ func TestParsingDownsamplingRatio(t *testing.T) { assert.Equal(t, f.FactoryConfig.DownsamplingRatio, 0.5) } +func TestPublishOpts(t *testing.T) { + f, err := NewFactory(defaultCfg()) + require.NoError(t, err) + + baseMetrics := metricstest.NewFactory(time.Second) + forkFactory := metricstest.NewFactory(time.Second) + metricsFactory := fork.New("internal", forkFactory, baseMetrics) + f.metricsFactory = metricsFactory + + // This method is called inside factory.Initialize method + f.publishOpts() + + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal." + downsamplingRatio, + Value: int(f.DownsamplingRatio), + }) + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal." + spanStorageType + "-" + f.SpanReaderType, + Value: 1, + }) +} + type errorCloseFactory struct { closeErr error } diff --git a/plugin/storage/grpc/README.md b/plugin/storage/grpc/README.md index fd8fd5da5a51..93876d785ef2 100644 --- a/plugin/storage/grpc/README.md +++ b/plugin/storage/grpc/README.md @@ -30,7 +30,7 @@ There are instructions on implementing a `go-plugin` server for non-Go languages [go-plugin non-go guide](https://github.com/hashicorp/go-plugin/blob/master/docs/guide-plugin-write-non-go.md). Take note of the required [health check service](https://github.com/hashicorp/go-plugin/blob/master/docs/guide-plugin-write-non-go.md#3-add-the-grpc-health-checking-service). -A Go plugin is a standalone application which calls `grpc.Serve(&plugin)` in its `main` function, where the `grpc` package +A Go plugin is a standalone application which calls `grpc.Serve(&pluginServices)` in its `main` function, where the `grpc` package is `github.com/jaegertracing/jaeger/plugin/storage/grpc`. ```go @@ -48,7 +48,10 @@ is `github.com/jaegertracing/jaeger/plugin/storage/grpc`. plugin := myStoragePlugin{} - grpc.Serve(&plugin) + grpc.Serve(&shared.PluginServices{ + Store: plugin, + ArchiveStore: plugin, + }) } ``` @@ -72,6 +75,23 @@ dependencies, you can also use `go.mod` to achieve the same goal of pinning your A simple plugin which uses the memstore storage implementation can be found in the `examples` directory of the top level of the Jaeger project. +To support archive storage a plugin must implement the ArchiveStoragePlugin interface of: + +```go +type ArchiveStoragePlugin interface { + ArchiveSpanReader() spanstore.Reader + ArchiveSpanWriter() spanstore.Writer +} +``` + +If you don't plan to implement archive storage simply do not fill `ArchiveStore` property of `shared.PluginServices`: + +```go +grpc.Serve(&shared.PluginServices{ + Store: plugin, +}) +``` + Running with a plugin --------------------- A plugin can be run using the `all-in-one` application within the top level `cmd` package of the Jaeger project. To do this diff --git a/plugin/storage/memory/factory.go b/plugin/storage/memory/factory.go index fa9fb4c5a06e..48d3d6bdcb2e 100644 --- a/plugin/storage/memory/factory.go +++ b/plugin/storage/memory/factory.go @@ -59,6 +59,8 @@ func (f *Factory) Initialize(metricsFactory metrics.Factory, logger *zap.Logger) f.metricsFactory, f.logger = metricsFactory, logger f.store = WithConfiguration(f.options.Configuration) logger.Info("Memory storage initialized", zap.Any("configuration", f.store.config)) + f.publishOpts() + return nil } @@ -76,3 +78,9 @@ func (f *Factory) CreateSpanWriter() (spanstore.Writer, error) { func (f *Factory) CreateDependencyReader() (dependencystore.Reader, error) { return f.store, nil } + +func (f *Factory) publishOpts() { + internalFactory := f.metricsFactory.Namespace(metrics.NSOptions{Name: "internal"}) + internalFactory.Gauge(metrics.Options{Name: limit}). + Update(int64(f.options.Configuration.MaxTraces)) +} diff --git a/plugin/storage/memory/factory_test.go b/plugin/storage/memory/factory_test.go index a7ecd5f33e7b..f024f2ff0a4a 100644 --- a/plugin/storage/memory/factory_test.go +++ b/plugin/storage/memory/factory_test.go @@ -17,8 +17,12 @@ package memory import ( "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/uber/jaeger-lib/metrics" + "github.com/uber/jaeger-lib/metrics/fork" + "github.com/uber/jaeger-lib/metrics/metricstest" "go.uber.org/zap" "github.com/jaegertracing/jaeger/pkg/config" @@ -29,7 +33,7 @@ var _ storage.Factory = new(Factory) func TestMemoryStorageFactory(t *testing.T) { f := NewFactory() - assert.NoError(t, f.Initialize(nil, zap.NewNop())) + assert.NoError(t, f.Initialize(metrics.NullFactory, zap.NewNop())) assert.NotNil(t, f.store) reader, err := f.CreateSpanReader() assert.NoError(t, err) @@ -56,3 +60,20 @@ func TestInitFromOptions(t *testing.T) { f.InitFromOptions(o) assert.Equal(t, o, f.options) } + +func TestPublishOpts(t *testing.T) { + f := NewFactory() + v, command := config.Viperize(f.AddFlags) + command.ParseFlags([]string{"--memory.max-traces=100"}) + f.InitFromViper(v) + + baseMetrics := metricstest.NewFactory(time.Second) + forkFactory := metricstest.NewFactory(time.Second) + metricsFactory := fork.New("internal", forkFactory, baseMetrics) + assert.NoError(t, f.Initialize(metricsFactory, zap.NewNop())) + + forkFactory.AssertGaugeMetrics(t, metricstest.ExpectedMetric{ + Name: "internal." + limit, + Value: f.options.Configuration.MaxTraces, + }) +}