[r95] patch with autocomplete_filtering flag and target_info PRs (#…

…2447)

* Add config param for autocomplete filtering (#2433)

* Add config param for autocomplete filtering

* Docs

* Fix

* Fix test

* Address comments

* Add job & instance labels to span metrics, a new target_info metrics, and the ability to customize dimension label mapping (#2261)

* add job, instance, target_info

* add test & gofmt

* fmt & test

* fix e2e test

* add custom dimension mapping

* updated test for custom mapping and changelog

* fmt

* changed the dimension mapping logic

* fix test

* update logic for job instance

* fix test & lint

* add service back and add gauge for target_info

* added overrides and docs

* undo config test change

* typo

* change traces_spanmetrics_target_info to just target_info

* lint

* fix overrides

* handle batches with different resource attributes

* redo counter

* refactor registry

* refactor label pair

* oops

* oops

* rebase mishap

* remove job instance by default

* refactor

* rebase hell

* count

* lint

* remove arbitary number

* traces_target_info

* comments

* sanitize labels for target info

* small refactor to improve perf

* rebased

* rebased

---------

Co-authored-by: Jennie Pham <[email protected]>

CHANGELOG.md

@@ -18,6 +18,7 @@
       concurrent_shards: 3
   ```
 * [FEATURE] Add support for `q` query param in `/api/v2/search/<tag.name>/values` to filter results based on a TraceQL query [#2253](https://github.com/grafana/tempo/pull/2253) (@mapno)
+To make use of filtering, configure `autocomplete_filtering_enabled`.
 * [FEATURE] Add a GRPC streaming endpoint for traceql search [#2366](https://github.com/grafana/tempo/pull/2366) (@joe-elliott)
 * [ENHANCEMENT] Add `scope` parameter to `/api/search/tags` [#2282](https://github.com/grafana/tempo/pull/2282) (@joe-elliott)
   Create new endpoint `/api/v2/search/tags` that returns all tags organized by scope.
@@ -86,6 +87,7 @@ storage:
 * [BUGFIX] Support float as duration like `{duration > 1.5s}` [#2304](https://github.com/grafana/tempo/pull/2304) (@ie-pham)
 * [ENHANCEMENT] Supports range operators for strings in TraceQL [#2321](https://github.com/grafana/tempo/pull/2321) (@ie-pham)
 * [ENHANCEMENT] Supports TraceQL in Vulture [#2321](https://github.com/grafana/tempo/pull/2321) (@ie-pham)
+* [FEATURE] Add job & instance labels to span metrics, a new target_info metrics, and custom dimension label mapping [#2261](https://github.com/grafana/tempo/pull/2261) (@ie-pham)
 
 ## v2.0.1 / 2023-03-03
 

cmd/tempo/app/app.go

@@ -51,8 +51,9 @@ var (
 		nil,
 	)
 
-	statFeatureEnabledAuth         = usagestats.NewInt("feature_enabled_auth_stats")
-	statFeatureEnabledMultitenancy = usagestats.NewInt("feature_enabled_multitenancy")
+	statFeatureEnabledAuth                  = usagestats.NewInt("feature_enabled_auth_stats")
+	statFeatureEnabledMultitenancy          = usagestats.NewInt("feature_enabled_multitenancy")
+	statFeatureAutocompleteFilteringEnabled = usagestats.NewInt("feature_enabled_autocomplete_filtering")
 )
 
 // App is the root datastructure.
@@ -100,6 +101,11 @@ func New(cfg Config) (*App, error) {
 		statFeatureEnabledMultitenancy.Set(1)
 	}
 
+	statFeatureAutocompleteFilteringEnabled.Set(0)
+	if cfg.AutocompleteFilteringEnabled {
+		statFeatureAutocompleteFilteringEnabled.Set(1)
+	}
+
 	app.setupAuthMiddleware()
 
 	if err := app.setupModuleManager(); err != nil {

cmd/tempo/app/config.go

@@ -27,12 +27,13 @@ import (
 
 // Config is the root config for App.
 type Config struct {
-	Target                 string `yaml:"target,omitempty"`
-	AuthEnabled            bool   `yaml:"auth_enabled,omitempty"`
-	MultitenancyEnabled    bool   `yaml:"multitenancy_enabled,omitempty"`
-	HTTPAPIPrefix          string `yaml:"http_api_prefix"`
-	UseOTelTracer          bool   `yaml:"use_otel_tracer,omitempty"`
-	EnableGoRuntimeMetrics bool   `yaml:"enable_go_runtime_metrics,omitempty"`
+	Target                       string `yaml:"target,omitempty"`
+	AuthEnabled                  bool   `yaml:"auth_enabled,omitempty"`
+	MultitenancyEnabled          bool   `yaml:"multitenancy_enabled,omitempty"`
+	HTTPAPIPrefix                string `yaml:"http_api_prefix"`
+	UseOTelTracer                bool   `yaml:"use_otel_tracer,omitempty"`
+	EnableGoRuntimeMetrics       bool   `yaml:"enable_go_runtime_metrics,omitempty"`
+	AutocompleteFilteringEnabled bool   `yaml:"autocomplete_filtering_enabled,omitempty"`
 
 	Server          server.Config           `yaml:"server,omitempty"`
 	InternalServer  internalserver.Config   `yaml:"internal_server,omitempty"`
@@ -67,6 +68,7 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	f.StringVar(&c.HTTPAPIPrefix, "http-api-prefix", "", "String prefix for all http api endpoints.")
 	f.BoolVar(&c.UseOTelTracer, "use-otel-tracer", false, "Set to true to replace the OpenTracing tracer with the OpenTelemetry tracer")
 	f.BoolVar(&c.EnableGoRuntimeMetrics, "enable-go-runtime-metrics", false, "Set to true to enable all Go runtime metrics")
+	f.BoolVar(&c.AutocompleteFilteringEnabled, "autocomplete-filtering.enabled", false, "Set to true to enable autocomplete filtering")
 
 	// Server settings
 	flagext.DefaultValues(&c.Server)

cmd/tempo/app/modules.go

@@ -186,6 +186,7 @@ func (t *App) initDistributor() (services.Service, error) {
 
 func (t *App) initIngester() (services.Service, error) {
 	t.cfg.Ingester.LifecyclerConfig.ListenPort = t.cfg.Server.GRPCListenPort
+	t.cfg.Ingester.AutocompleteFilteringEnabled = t.cfg.AutocompleteFilteringEnabled
 	ingester, err := ingester.New(t.cfg.Ingester, t.store, t.overrides, prometheus.DefaultRegisterer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ingester: %w", err)

docs/sources/tempo/configuration/_index.md

@@ -64,6 +64,11 @@ Tempo uses the Weaveworks/common server. For more information on configuration o
 ```yaml
 # Optional. Setting to true enables multitenancy and requires X-Scope-OrgID header on all requests.
 [multitenancy_enabled: <bool> | default = false]
+
+# Optional. Setting to true enables query filtering in tag value search API `/api/v2/search/<tag>/values`.
+# If filtering is enabled, the API accepts a query parameter `q` containing a TraceQL query,
+# and returns only tag values that match the query.
+[autocomplete_filtering_enabled: <bool> | default = false]
 
 # Optional. String prefix for all http api endpoints. Must include beginning slash.
 [http_api_prefix: <string>]
@@ -297,6 +302,14 @@ metrics_generator:
             # the metrics if present.
             [dimensions: <list of string>]
 
+            # Custom labeling of dimensions is possible via a list of maps consisting of 
+            # "name" <string>, "source_labels" <list of string>, "join" <string> 
+            # "name" appears in the metrics, "source_labels" are the actual
+            # attributes that will make up the value of the label and "join" is the
+            # separator if multiple source_labels are provided
+            [MetricsGeneratorProcessorSpanMetricsDimensionMappings: <list of map>]
+            # Enable target_info metrics
+            [MetricsGeneratorProcessorSpanMetricsEnableTargetInfo: <bool>]
             # Attribute Key to multiply span metrics
             [span_multiplier_key: <string> | default = ""]
 
@@ -1243,6 +1256,9 @@ overrides:
     # Allowed keys for intrinsic dimensions are: service, span_name, span_kind, status_code, and status_message.
     [metrics_generator_processor_span_metrics_intrinsic_dimensions: <map string to bool>]
     [metrics_generator_processor_span_metrics_dimensions: <list of string>]
+    [MetricsGeneratorProcessorSpanMetricsDimensionMappings: <list of map>]
+    # Enable target_info metrics
+    [MetricsGeneratorProcessorSpanMetricsEnableTargetInfo: <bool>]
 
     # Maximum number of active series in the registry, per instance of the metrics-generator. A
     # value of 0 disables this check.

docs/sources/tempo/metrics-generator/span_metrics.md

@@ -53,10 +53,14 @@ The following metrics are exported:
 In Tempo 1.4 and 1.4.1, the histogram metric was called `traces_spanmetrics_duration_seconds`. This was changed later to be consistent with the metrics generated by the Grafana Agent and the OpenTelemetry Collector.
 {{% /admonition %}}
 
-By default, the metrics processor adds the following labels to each metric: `service`, `span_name`, `span_kind`, `status_code`, `status_message`.
+By default, the metrics processor adds the following labels to each metric: `service`, `span_name`, `span_kind`, `status_code`, `status_message`, `job`, `instance`.
 Additional user defined labels can be created using the [`dimensions` configuration option]({{< relref "../configuration/#metrics-generator" >}}).
 When a configured dimension collides with one of the default labels (e.g. `status_code`), the label for the respective dimension is prefixed with double underscore (i.e. `__status_code`).
 
+Custom labeling of dimensions is also supported using the [`dimension_mapping` configuration option]({{< relref "../configuration/#metrics-generator" >}}).
+
+An optional metric called `traces_target_info` using all resource level attributes as dimensions can be enabled in the [`enable_target_info` configuration option]({{< relref "../configuration/#metrics-generator" >}}).
+
 If you use ratio based sampler you can use custom sampler below to not lose metric information, you also need to set `metrics_generator.processor.span_metrics.span_multiplier_key` to `"X-SampleRatio"`
 
 ```go

go.mod

@@ -161,7 +161,7 @@ require (
 	github.com/google/btree v1.0.1 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
-	github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 // indirect
+	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
 	github.com/hashicorp/consul/api v1.15.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect

go.sum

@@ -488,8 +488,8 @@ github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9 h1:WB3bGH2f1UN6
 github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU=
 github.com/grafana/memberlist v0.3.1-0.20220708130638-bd88e10a3d91 h1:/NipyHnOmvRsVzj81j2qE0VxsvsqhOB0f4vJIhk2qCQ=
 github.com/grafana/memberlist v0.3.1-0.20220708130638-bd88e10a3d91/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
-github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 h1:A3dhViTeFDSQcGOXuUi6ukCQSMyDtDISBp2z6OOo2YM=
-github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
+github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA=
+github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=

integration/e2e/api_test.go

@@ -22,7 +22,7 @@ func TestSearchTagValuesV2(t *testing.T) {
 	defer s.Close()
 
 	require.NoError(t, util.CopyFileToSharedDir(s, configAllInOneLocal, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
+	tempo := util.NewTempoAllInOne("-autocomplete-filtering.enabled=true")
 	require.NoError(t, s.StartAndWaitReady(tempo))
 
 	jaegerClient, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250))

integration/e2e/config-metrics-generator-targetinfo.yaml

@@ -0,0 +1,47 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    jaeger:
+      protocols:
+        grpc:
+  log_received_spans:
+    enabled: true
+
+ingester:
+  lifecycler:
+    ring:
+      replication_factor: 1
+
+metrics_generator:
+  processor:
+    service_graphs:
+      histogram_buckets: [1, 2]  # seconds
+    span_metrics:
+      histogram_buckets: [1, 2]
+  registry:
+    collection_interval: 1s
+  storage:
+    path: /var/tempo
+    remote_write:
+      - url: http://tempo_e2e-prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo
+
+memberlist:
+  bind_port: 7946
+  join_members:
+    - tempo_e2e-distributor:7946
+    - tempo_e2e-ingester-1:7946
+    - tempo_e2e-metrics-generator:7946
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics]
+  metrics_generator_max_active_series: 1000
+  metrics_generator_processor_span_metrics_enable_target_info: true  # seconds