Include metrics for configured limit overrides (#1089)

* Include metrics for configured limit overrides

* Format config docs, and fix wording

* Update changelog

* Updates based on PR feedback

* Include metrics for default limits

* Implement Collector pattern

* Move default limits metric to Limits struct

* Use constants for metric names and unify

CHANGELOG.md

@@ -5,6 +5,7 @@
 * [ENHANCEMENT] Add middleware to compress frontend HTTP responses with gzip if requested [#1080](https://github.com/grafana/tempo/pull/1080) (@kvrhdn, @zalegrala)
 * [ENHANCEMENT] Allow query disablement in vulture [#1117](https://github.com/grafana/tempo/pull/1117) (@zalegrala)
 * [ENHANCEMENT] Improve memory efficiency of compaction and block cutting. [#1121](https://github.com/grafana/tempo/pull/1121) (@joe-elliott)
+* [ENHANCEMENT] Include metrics for configured limit overrides and defaults: tempo_limits_overrides, tempo_limits_defaults [#1089](https://github.com/grafana/tempo/pull/1089) (@zalegrala)
 * [BUGFIX] Fix defaults for MaxBytesPerTrace (ingester.max-bytes-per-trace) and MaxSearchBytesPerTrace (ingester.max-search-bytes-per-trace) (@bitprocessor)
 * [BUGFIX] Ignore empty objects during compaction [#1113](https://github.com/grafana/tempo/pull/1113) (@mdisibio)
 

cmd/tempo/app/modules.go

@@ -95,6 +95,12 @@ func (t *App) initOverrides() (services.Service, error) {
 	}
 	t.overrides = overrides
 
+	prometheus.MustRegister(&t.cfg.LimitsConfig)
+
+	if t.cfg.LimitsConfig.PerTenantOverrideConfig != "" {
+		prometheus.MustRegister(t.overrides)
+	}
+
 	return t.overrides, nil
 }
 

docs/tempo/website/configuration/_index.md

@@ -702,7 +702,8 @@ memberlist:
 ```
 
 ## Overrides
-Tempo provides a overrides module for user to set global or per-tenant override settings.   
+
+Tempo provides an overrides module for users to set global or per-tenant override settings.
 **Currenly only ingestion limits can be overridden.**
 
 ### Ingestion limits

modules/overrides/limits.go

@@ -3,6 +3,7 @@ package overrides
 import (
 	"flag"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 )
 
@@ -18,6 +19,24 @@ const (
 	ErrorPrefixTraceTooLarge = "TRACE_TOO_LARGE:"
 	// ErrorPrefixRateLimited is used to flag batches that have exceeded the spans/second of the tenant
 	ErrorPrefixRateLimited = "RATE_LIMITED:"
+
+	// metrics
+	MetricMaxLocalTracesPerUser   = "max_local_traces_per_user"
+	MetricMaxGlobalTracesPerUser  = "max_global_traces_per_user"
+	MetricMaxBytesPerTrace        = "max_bytes_per_trace"
+	MetricMaxSearchBytesPerTrace  = "max_search_bytes_per_trace"
+	MetricIngestionRateLimitBytes = "ingestion_rate_limit_bytes"
+	MetricIngestionBurstSizeBytes = "ingestion_burst_size_bytes"
+	MetricBlockRetention          = "block_retention"
+)
+
+var (
+	metricLimitsDesc = prometheus.NewDesc(
+		"tempo_limits_defaults",
+		"Default resource limits",
+		[]string{"limit_name"},
+		nil,
+	)
 )
 
 // Limits describe all the limits for users; can be used to describe global default
@@ -60,3 +79,17 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	_ = l.PerTenantOverridePeriod.Set("10s")
 	f.Var(&l.PerTenantOverridePeriod, "limits.per-user-override-period", "Period with this to reload the overrides.")
 }
+
+func (l *Limits) Describe(ch chan<- *prometheus.Desc) {
+	ch <- metricLimitsDesc
+}
+
+func (l *Limits) Collect(ch chan<- prometheus.Metric) {
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.MaxLocalTracesPerUser), MetricMaxLocalTracesPerUser)
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.MaxGlobalTracesPerUser), MetricMaxGlobalTracesPerUser)
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.MaxBytesPerTrace), MetricMaxBytesPerTrace)
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.MaxSearchBytesPerTrace), MetricMaxSearchBytesPerTrace)
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.IngestionRateLimitBytes), MetricIngestionRateLimitBytes)
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.IngestionBurstSizeBytes), MetricIngestionBurstSizeBytes)
+	ch <- prometheus.MustNewConstMetric(metricLimitsDesc, prometheus.GaugeValue, float64(l.BlockRetention), MetricBlockRetention)
+}

modules/overrides/overrides.go

@@ -17,6 +17,15 @@ import (
 
 const wildcardTenant = "*"
 
+var (
+	metricOverridesLimitsDesc = prometheus.NewDesc(
+		"tempo_limits_overrides",
+		"Resource limit overrides applied to tenants",
+		[]string{"limit_name", "user"},
+		nil,
+	)
+)
+
 // perTenantOverrides represents the overrides config file
 type perTenantOverrides struct {
 	TenantLimits map[string]*Limits `yaml:"overrides"`
@@ -233,22 +242,22 @@ func (o *Overrides) MaxSearchBytesPerTrace(userID string) int {
 	return o.getOverridesForUser(userID).MaxSearchBytesPerTrace
 }
 
-// IngestionRateLimitBytes is the number of spans per second allowed for this tenant
+// IngestionRateLimitBytes is the number of spans per second allowed for this tenant.
 func (o *Overrides) IngestionRateLimitBytes(userID string) float64 {
 	return float64(o.getOverridesForUser(userID).IngestionRateLimitBytes)
 }
 
-// IngestionBurstSizeBytes is the burst size in spans allowed for this tenant
+// IngestionBurstSizeBytes is the burst size in spans allowed for this tenant.
 func (o *Overrides) IngestionBurstSizeBytes(userID string) int {
 	return o.getOverridesForUser(userID).IngestionBurstSizeBytes
 }
 
-// SearchTagsAllowList is the list of tags to be extracted for search, for this tenant
+// SearchTagsAllowList is the list of tags to be extracted for search, for this tenant.
 func (o *Overrides) SearchTagsAllowList(userID string) map[string]struct{} {
 	return o.getOverridesForUser(userID).SearchTagsAllowList.GetMap()
 }
 
-// BlockRetention is the duration of the block retention for this tenant
+// BlockRetention is the duration of the block retention for this tenant.
 func (o *Overrides) BlockRetention(userID string) time.Duration {
 	return time.Duration(o.getOverridesForUser(userID).BlockRetention)
 }
@@ -264,8 +273,29 @@ func (o *Overrides) getOverridesForUser(userID string) *Limits {
 		if l != nil {
 			return l
 		}
-
 	}
 
 	return o.defaultLimits
 }
+
+func (o *Overrides) Describe(ch chan<- *prometheus.Desc) {
+	ch <- metricOverridesLimitsDesc
+}
+
+func (o *Overrides) Collect(ch chan<- prometheus.Metric) {
+	overrides := o.tenantOverrides()
+	if overrides == nil {
+		return
+	}
+
+	for tenant, limits := range overrides.TenantLimits {
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.MaxLocalTracesPerUser), MetricMaxLocalTracesPerUser, tenant)
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.MaxGlobalTracesPerUser), MetricMaxGlobalTracesPerUser, tenant)
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.MaxBytesPerTrace), MetricMaxBytesPerTrace, tenant)
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.MaxSearchBytesPerTrace), MetricMaxSearchBytesPerTrace, tenant)
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.IngestionRateLimitBytes), MetricIngestionRateLimitBytes, tenant)
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.IngestionBurstSizeBytes), MetricIngestionBurstSizeBytes, tenant)
+		ch <- prometheus.MustNewConstMetric(metricOverridesLimitsDesc, prometheus.GaugeValue, float64(limits.BlockRetention), MetricBlockRetention, tenant)
+
+	}
+}