diff --git a/cmd/statshouse/worker.go b/cmd/statshouse/worker.go index 2b4277d70..fa1b091b7 100644 --- a/cmd/statshouse/worker.go +++ b/cmd/statshouse/worker.go @@ -59,10 +59,6 @@ func (w *worker) HandleMetrics(args data_model.HandlerArgs) (h data_model.Mapped w.fillTime(args, &h) metaOk := w.fillMetricMeta(args, &h) if metaOk { - h.Key.Metric = h.MetricMeta.MetricID - if !h.MetricMeta.Visible { - h.IngestionStatus = format.TagValueIDSrcIngestionStatusErrMetricInvisible - } done = w.mapper.Map(args, h.MetricMeta, &h) } else { w.mapper.MapEnvironment(args.MetricBytes, &h) @@ -95,11 +91,21 @@ func (w *worker) fillMetricMeta(args data_model.HandlerArgs, h *data_model.Mappe metricMeta := w.metricStorage.GetMetaMetricByNameBytes(metric.Name) if metricMeta != nil { h.MetricMeta = metricMeta + h.Key.Metric = metricMeta.MetricID + if !metricMeta.Visible { + h.IngestionStatus = format.TagValueIDSrcIngestionStatusErrMetricInvisible + return false + } return true } - metricMeta = format.BuiltinMetricAllowedToReceive[string(metric.Name)] + metricMeta = format.BuiltinMetricByName[string(metric.Name)] if metricMeta != nil { h.MetricMeta = metricMeta + h.Key.Metric = metricMeta.MetricID + if !metricMeta.BuiltinAllowedToReceive { + h.IngestionStatus = format.TagValueIDSrcIngestionStatusErrMetricBuiltin + return false + } return true } diff --git a/internal/format/builtin.go b/internal/format/builtin.go index 904dfa2c0..f62fc4634 100644 --- a/internal/format/builtin.go +++ b/internal/format/builtin.go @@ -41,10 +41,6 @@ const ( var ( BuiltinMetricByName map[string]*MetricMetaValue - // list of built-in metrics which can be sent as normal metric. Used by API and prometheus exporter - // Description: "-" marks tags used in incompatible way in the past. We should not reuse such tags, because there would be garbage in historic data. - BuiltinMetricAllowedToReceive map[string]*MetricMetaValue - BuiltinMetrics = map[int32]*MetricMetaValue{ BuiltinMetricIDAgentSamplingFactor: BuiltinMetricMetaAgentSamplingFactor, BuiltinMetricIDAggBucketReceiveDelaySec: BuiltinMetricMetaAggBucketReceiveDelaySec, @@ -160,6 +156,7 @@ var ( BuiltinMetricIDBudgetOwner: BuiltinMetricMetaBudgetOwner, } + // this set is very small, and we do not want to set Visible property for hunderds of metrics builtinMetricsInvisible = map[int32]bool{ BuiltinMetricIDBudgetHost: true, BuiltinMetricIDBudgetAggregatorHost: true, @@ -167,186 +164,6 @@ var ( BuiltinMetricIDBudgetOwner: true, } - // API and metadata sends this metrics via local statshouse instance - builtinMetricsAllowedToReceive = map[int32]bool{ - BuiltinMetricIDTimingErrors: true, - BuiltinMetricIDPromScrapeTime: true, - BuiltinMetricIDAPIBRS: true, - BuiltinMetricIDAPIServiceTime: true, - BuiltinMetricIDAPIResponseTime: true, - BuiltinMetricIDUsageMemory: true, - BuiltinMetricIDUsageCPU: true, - BuiltinMetricIDAPIActiveQueries: true, - BuiltinMetricIDAPISelectRows: true, - BuiltinMetricIDAPISelectBytes: true, - BuiltinMetricIDAPISelectDuration: true, - BuiltinMetricIDSystemMetricScrapeDuration: true, - BuiltinMetricIDMetaServiceTime: true, - BuiltinMetricIDMetaClientWaits: true, - BuiltinMetricIDAPIMetricUsage: true, - BuiltinMetricIDHeartbeatVersion: true, - BuiltinMetricIDUIErrors: true, - BuiltinMetricIDStatsHouseErrors: true, - BuiltinMetricIDPromQLEngineTime: true, - BuiltinMetricIDAPICacheHit: true, - BuiltinMetricIDAPICacheBytesAlloc: true, - BuiltinMetricIDAPICacheBytesFree: true, - BuiltinMetricIDAPICacheBytesTotal: true, - BuiltinMetricIDAPICacheAgeEvict: true, - BuiltinMetricIDAPICacheAgeTotal: true, - BuiltinMetricIDAPIBufferBytesAlloc: true, - BuiltinMetricIDAPIBufferBytesFree: true, - BuiltinMetricIDAPIBufferBytesTotal: true, - BuiltinMetricIDAutoCreateMetric: true, - BuiltinMetricIDGCDuration: true, - BuiltinMetricIDProxyAcceptHandshakeError: true, - BuiltinMetricIDProxyVmSize: true, - BuiltinMetricIDProxyVmRSS: true, - BuiltinMetricIDProxyHeapAlloc: true, - BuiltinMetricIDProxyHeapSys: true, - BuiltinMetricIDProxyHeapIdle: true, - BuiltinMetricIDProxyHeapInuse: true, - BuiltinMetricIDApiVmSize: true, - BuiltinMetricIDApiVmRSS: true, - BuiltinMetricIDApiHeapAlloc: true, - BuiltinMetricIDApiHeapSys: true, - BuiltinMetricIDApiHeapIdle: true, - BuiltinMetricIDApiHeapInuse: true, - BuiltinMetricIDClientWriteError: true, - } - - builtinMetricsNoSamplingAgent = map[int32]bool{ - BuiltinMetricIDAgentMapping: true, - BuiltinMetricIDJournalVersions: true, - BuiltinMetricIDAgentReceivedPacketSize: true, - BuiltinMetricIDAgentReceivedBatchSize: true, - BuiltinMetricIDHeartbeatVersion: true, - BuiltinMetricIDHeartbeatArgs: true, - BuiltinMetricIDAgentDiskCacheErrors: true, - BuiltinMetricIDTimingErrors: true, - BuiltinMetricIDUsageMemory: true, - BuiltinMetricIDUsageCPU: true, - - BuiltinMetricIDCPUUsage: true, - BuiltinMetricIDMemUsage: true, - BuiltinMetricIDProcessCreated: true, - BuiltinMetricIDProcessRunning: true, - BuiltinMetricIDSystemUptime: true, - BuiltinMetricIDPSICPU: true, - BuiltinMetricIDPSIMem: true, - BuiltinMetricIDPSIIO: true, - BuiltinMetricIDNetBandwidth: true, - BuiltinMetricIDNetPacket: true, - BuiltinMetricIDNetError: true, - BuiltinMetricIDDiskUsage: true, - BuiltinMetricIDINodeUsage: true, - BuiltinMetricIDTCPSocketStatus: true, - BuiltinMetricIDTCPSocketMemory: true, - BuiltinMetricIDSocketMemory: true, - BuiltinMetricIDSocketUsed: true, - BuiltinMetricIDSoftIRQ: true, - BuiltinMetricIDIRQ: true, - BuiltinMetricIDContextSwitch: true, - BuiltinMetricIDWriteback: true, - } - - MetricsWithAgentEnvRouteArch = map[int32]bool{ - BuiltinMetricIDAgentDiskCacheErrors: true, - BuiltinMetricIDTimingErrors: true, - BuiltinMetricIDAgentMapping: true, - BuiltinMetricIDAutoConfig: true, // also passed through ingress proxies - BuiltinMetricIDJournalVersions: true, - BuiltinMetricIDTLByteSizePerInflightType: true, - BuiltinMetricIDIngestionStatus: true, - BuiltinMetricIDAgentReceivedBatchSize: true, - BuiltinMetricIDAgentReceivedPacketSize: true, - BuiltinMetricIDAggSizeCompressed: true, - BuiltinMetricIDAggSizeUncompressed: true, - BuiltinMetricIDAggBucketReceiveDelaySec: true, - BuiltinMetricIDAggBucketAggregateTimeSec: true, - BuiltinMetricIDAggAdditionsToEstimator: true, - BuiltinMetricIDAgentHistoricQueueSize: true, - BuiltinMetricIDVersions: true, - BuiltinMetricIDAggKeepAlive: true, - BuiltinMetricIDAggMappingCreated: true, - BuiltinMetricIDUsageMemory: true, - BuiltinMetricIDUsageCPU: true, - BuiltinMetricIDHeartbeatVersion: true, - BuiltinMetricIDHeartbeatArgs: true, - BuiltinMetricIDAgentUDPReceiveBufferSize: true, - BuiltinMetricIDAgentDiskCacheSize: true, - BuiltinMetricIDSrcTestConnection: true, - BuiltinMetricIDAgentAggregatorTimeDiff: true, - BuiltinMetricIDSrcSamplingMetricCount: true, - BuiltinMetricIDSrcSamplingSizeBytes: true, - BuiltinMetricIDStatsHouseErrors: true, - BuiltinMetricIDSrcSamplingBudget: true, - BuiltinMetricIDSrcSamplingGroupBudget: true, - BuiltinMetricIDRestartTimings: true, - BuiltinMetricIDGCDuration: true, - BuiltinMetricIDAgentTimings: true, - BuiltinMetricIDAggBucketInfo: true, - } - - metricsWithoutAggregatorID = map[int32]bool{ - BuiltinMetricIDTLByteSizePerInflightType: true, - BuiltinMetricIDIngestionStatus: true, - BuiltinMetricIDAgentDiskCacheErrors: true, - BuiltinMetricIDAgentReceivedBatchSize: true, - BuiltinMetricIDAgentMapping: true, - BuiltinMetricIDAgentReceivedPacketSize: true, - BuiltinMetricIDBadges: true, - BuiltinMetricIDPromScrapeTime: true, - BuiltinMetricIDGeneratorConstCounter: true, - BuiltinMetricIDGeneratorSinCounter: true, - BuiltinMetricIDAPIBRS: true, - BuiltinMetricIDAPISelectRows: true, - BuiltinMetricIDAPISelectBytes: true, - BuiltinMetricIDAPISelectDuration: true, - BuiltinMetricIDAPIServiceTime: true, - BuiltinMetricIDAPIResponseTime: true, - BuiltinMetricIDAPIActiveQueries: true, - BuiltinMetricIDBudgetHost: true, - BuiltinMetricIDBudgetAggregatorHost: true, - BuiltinMetricIDSystemMetricScrapeDuration: true, - BuiltinMetricIDAgentUDPReceiveBufferSize: true, - BuiltinMetricIDAgentDiskCacheSize: true, - BuiltinMetricIDAPIMetricUsage: true, - BuiltinMetricIDSrcSamplingMetricCount: true, - BuiltinMetricIDSrcSamplingSizeBytes: true, - BuiltinMetricIDSrcSamplingBudget: true, - BuiltinMetricIDSrcSamplingGroupBudget: true, - BuiltinMetricIDUIErrors: true, - BuiltinMetricIDStatsHouseErrors: true, - BuiltinMetricIDPromQLEngineTime: true, - BuiltinMetricIDAPICacheBytesAlloc: true, - BuiltinMetricIDAPICacheBytesFree: true, - BuiltinMetricIDAPICacheBytesTotal: true, - BuiltinMetricIDAPICacheAgeEvict: true, - BuiltinMetricIDAPICacheAgeTotal: true, - BuiltinMetricIDAPIBufferBytesAlloc: true, - BuiltinMetricIDAPIBufferBytesFree: true, - BuiltinMetricIDAPIBufferBytesTotal: true, - BuiltinMetricIDRestartTimings: true, - BuiltinMetricIDGCDuration: true, - BuiltinMetricIDProxyAcceptHandshakeError: true, - BuiltinMetricIDProxyVmSize: true, - BuiltinMetricIDProxyVmRSS: true, - BuiltinMetricIDProxyHeapAlloc: true, - BuiltinMetricIDProxyHeapSys: true, - BuiltinMetricIDProxyHeapIdle: true, - BuiltinMetricIDProxyHeapInuse: true, - BuiltinMetricIDApiVmSize: true, - BuiltinMetricIDApiVmRSS: true, - BuiltinMetricIDApiHeapAlloc: true, - BuiltinMetricIDApiHeapSys: true, - BuiltinMetricIDApiHeapIdle: true, - BuiltinMetricIDApiHeapInuse: true, - BuiltinMetricIDClientWriteError: true, - BuiltinMetricIDAgentTimings: true, - BuiltinMetricIDBudgetOwner: true, - } - // BuiltInGroupDefault can be overridden by journal, don't use directly BuiltInGroupDefault = map[int32]*MetricsGroup{ BuiltinGroupIDDefault: { @@ -373,36 +190,6 @@ var ( Weight: 1, }, } - - versionToValue = map[int32]string{ - 1: "v1", - 2: "v2", - } - - secondsToValue = map[int32]string{ - 1: "1s", - 5: "5s", - 15: "15s", - 60: "1m", - 300: "5m", - 900: "15m", - 3600: "1h", - 14400: "4h", - 86400: "24h", - 604800: "7d", - 2678400: "1M", - } - - apiCacheEvictionReason = map[int32]string{ - 1: "stale", // known to be stale - 2: "LRU", // evicted to free up memory - 3: "update", // evicted by more recent load - } - - apiBufferKind = map[int32]string{ - 1: "pool", // "sync.Pool", allocated buffer is subject for reuse (good) - 2: "heap", // large buffer won't be reused (bad, should not happen) - } ) func TagIDTagToTagID(tagIDTag int32) string { @@ -433,8 +220,10 @@ func init() { v.Group = BuiltInGroupDefault[BuiltinGroupIDHost] v.Sharding = []MetricSharding{{Strategy: ShardByMetric}} BuiltinMetrics[k] = v - builtinMetricsAllowedToReceive[k] = true - metricsWithoutAggregatorID[k] = true + // v.NoSampleAgent = false + v.BuiltinAllowedToReceive = true + // v.WithAgentEnvRouteArch = false + // v.WithAggregatorID = false } for i := 0; i < NewMaxTags; i++ { name := strconv.Itoa(i) @@ -454,7 +243,6 @@ func init() { tagIDTag2TagID[TagIDShift-2] = tagStringForUI + " " + HostTagID // for UI only BuiltinMetricByName = make(map[string]*MetricMetaValue, len(BuiltinMetrics)) - BuiltinMetricAllowedToReceive = make(map[string]*MetricMetaValue, len(BuiltinMetrics)) for id, m := range BuiltinMetrics { m.MetricID = id if m.GroupID == 0 { @@ -467,9 +255,6 @@ func init() { BuiltinMetricByName[m.Name] = m - if builtinMetricsAllowedToReceive[m.MetricID] { - BuiltinMetricAllowedToReceive[m.Name] = m - } if id == BuiltinMetricIDIngestionStatus || id == BuiltinMetricIDAggMappingCreated { m.Tags = append([]MetricMetaTag{{Description: "environment"}}, m.Tags...) } else { @@ -478,7 +263,7 @@ func init() { for len(m.Tags) < MaxTags { m.Tags = append(m.Tags, MetricMetaTag{Description: "-"}) } - if !metricsWithoutAggregatorID[id] { + if m.WithAggregatorID { m.Tags[AggHostTag] = MetricMetaTag{Description: "aggregator_host"} m.Tags[AggShardTag] = MetricMetaTag{Description: "aggregator_shard", Raw: true} m.Tags[AggReplicaTag] = MetricMetaTag{Description: "aggregator_replica", Raw: true} @@ -491,7 +276,7 @@ func init() { m.Tags[HostOwnerTag] = MetricMetaTag{Description: "owner"} } - if MetricsWithAgentEnvRouteArch[id] { + if m.WithAgentEnvRouteArch { m.Tags[RouteTag] = MetricMetaTag{Description: "route", ValueComments: convertToValueComments(routeToValue)} m.Tags[AgentEnvTag] = MetricMetaTag{ Description: "statshouse_env", diff --git a/internal/format/builtin_metrics.go b/internal/format/builtin_metrics.go index b45f7c3e5..c8c562b48 100644 --- a/internal/format/builtin_metrics.go +++ b/internal/format/builtin_metrics.go @@ -9,6 +9,7 @@ package format import "github.com/mailru/easyjson/opt" // metric metas for builtin metrics are accessible directly without search in map +// Description: "-" marks tags used in incompatible way in the past. We should not reuse such tags, because there would be garbage in historic data. const BuiltinMetricIDAgentSamplingFactor = -1 @@ -19,6 +20,10 @@ var BuiltinMetricMetaAgentSamplingFactor = &MetricMetaValue{ Calculated by agent from scratch every second to fit all collected data into network budget. Count of this metric is proportional to # of clients who set it in particular second. Set only if greater than 1.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "metric", IsMetric: true, @@ -39,7 +44,11 @@ var BuiltinMetricMetaAggBucketReceiveDelaySec = &MetricMetaValue{ Description: `Difference between timestamp of received bucket and aggregator wall clock. Count of this metric is # of agents who sent this second (per replica*shard), and they do it every second to keep this metric stable. Set by aggregator.`, - MetricType: MetricSecond, + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -65,10 +74,14 @@ Set by aggregator.`, const BuiltinMetricIDAggInsertSize = -3 // If all contributors come on time, count will be 1 (per shard). If some come through historic conveyor, can be larger. var BuiltinMetricMetaAggInsertSize = &MetricMetaValue{ - Name: "__agg_insert_size", - Kind: MetricKindValue, - Description: "Size of aggregated bucket inserted into clickhouse. Written when second is inserted, which can be much later.", - MetricType: MetricByte, + Name: "__agg_insert_size", + Kind: MetricKindValue, + Description: "Size of aggregated bucket inserted into clickhouse. Written when second is inserted, which can be much later.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -89,10 +102,14 @@ var BuiltinMetricMetaAggInsertSize = &MetricMetaValue{ const BuiltinMetricIDTLByteSizePerInflightType = -4 var BuiltinMetricMetaTLByteSizePerInflightType = &MetricMetaValue{ - Name: "__src_tl_byte_size_per_inflight_type", - Kind: MetricKindValue, - Description: "Approximate uncompressed byte size of various parts of TL representation of time bucket.\nSet by agent.", - MetricType: MetricByte, + Name: "__src_tl_byte_size_per_inflight_type", + Kind: MetricKindValue, + Description: "Approximate uncompressed byte size of various parts of TL representation of time bucket.\nSet by agent.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "inflight_type", ValueComments: convertToValueComments(insertKindToValue), @@ -101,9 +118,13 @@ var BuiltinMetricMetaTLByteSizePerInflightType = &MetricMetaValue{ const BuiltinMetricIDAggKeepAlive = -5 // How many keep-alive were among contributors var BuiltinMetricMetaAggKeepAlive = &MetricMetaValue{ - Name: "__agg_keep_alive", - Kind: MetricKindCounter, - Description: "Number of keep-alive empty inserts (which follow normal insert conveyor) in aggregated bucket.\nSet by aggregator.", + Name: "__agg_keep_alive", + Kind: MetricKindCounter, + Description: "Number of keep-alive empty inserts (which follow normal insert conveyor) in aggregated bucket.\nSet by aggregator.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -118,10 +139,14 @@ var BuiltinMetricMetaAggKeepAlive = &MetricMetaValue{ const BuiltinMetricIDAggSizeCompressed = -6 var BuiltinMetricMetaAggSizeCompressed = &MetricMetaValue{ - Name: "__agg_size_compressed", - Kind: MetricKindValue, - Description: "Compressed size of bucket received from agent (size of raw TL request).\nSet by aggregator.", - MetricType: MetricByte, + Name: "__agg_size_compressed", + Kind: MetricKindValue, + Description: "Compressed size of bucket received from agent (size of raw TL request).\nSet by aggregator.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -142,10 +167,14 @@ var BuiltinMetricMetaAggSizeCompressed = &MetricMetaValue{ const BuiltinMetricIDAggSizeUncompressed = -7 var BuiltinMetricMetaAggSizeUncompressed = &MetricMetaValue{ - Name: "__agg_size_uncompressed", - Kind: MetricKindValue, - Description: "Uncompressed size of bucket received from agent.\nSet by aggregator.", - MetricType: MetricByte, + Name: "__agg_size_uncompressed", + Kind: MetricKindValue, + Description: "Uncompressed size of bucket received from agent.\nSet by aggregator.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -169,6 +198,10 @@ var BuiltinMetricMetaAggAdditionsToEstimator = &MetricMetaValue{ Kind: MetricKindValue, Description: `How many unique metric-tag combinations were inserted into aggregation bucket. Set by aggregator. Max(value)@host shows host responsible for most combinations, and is very order-dependent.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -196,7 +229,11 @@ Linear interpolation between previous hour and value collected so far for this h Steps of interpolation can be visible on graph. Each aggregator writes value on every insert to particular second, multiplied by # of aggregator shards. So avg() of this metric shows estimated full cardinality with or without grouping by aggregator.`, - Resolution: 60, + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -218,6 +255,10 @@ var BuiltinMetricMetaAggSamplingFactor = &MetricMetaValue{ Description: `Sample factor selected by aggregator. Calculated by aggregator from scratch every second to fit all collected data into clickhouse insert budget. Set only if greater than 1.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -246,7 +287,11 @@ Most errors are due to various data format violation. Some, like 'err_map_per_metric_queue_overload', 'err_map_tag_value', 'err_map_tag_value_cached' indicate tag mapping subsystem slowdowns or errors. This metric uses sampling budgets of metric it refers to, so flooding by errors cannot affect other metrics. 'err_*_utf8'' statuses store original string value in hex.`, - StringTopDescription: "string_value", + StringTopDescription: "string_value", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "metric", IsMetric: true, @@ -284,6 +329,7 @@ This metric uses sampling budgets of metric it refers to, so flooding by errors TagValueIDSrcIngestionStatusErrShardingFailed: "err_sharding_failed", TagValueIDSrcIngestionStatusWarnTimestampClampedPast: "warn_timestamp_clamped_past", TagValueIDSrcIngestionStatusWarnTimestampClampedFuture: "warn_timestamp_clamped_future", + TagValueIDSrcIngestionStatusErrMetricBuiltin: "err_metric_builtin", }), }, { Description: "tag_id", @@ -298,10 +344,14 @@ This metric uses sampling budgets of metric it refers to, so flooding by errors const BuiltinMetricIDAggInsertTime = -12 var BuiltinMetricMetaAggInsertTime = &MetricMetaValue{ - Name: "__agg_insert_time", - Kind: MetricKindValue, - Description: "Time inserting this second into clickhouse took. Written when second is inserted, which can be much later.", - MetricType: MetricSecond, + Name: "__agg_insert_time", + Kind: MetricKindValue, + Description: "Time inserting this second into clickhouse took. Written when second is inserted, which can be much later.", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -327,7 +377,11 @@ var BuiltinMetricMetaAggHistoricBucketsWaiting = &MetricMetaValue{ Kind: MetricKindValue, Description: `Time difference of historic seconds (several per contributor) waiting to be inserted via historic conveyor. Count is number of such seconds waiting.`, - MetricType: MetricSecond, + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -350,7 +404,11 @@ var BuiltinMetricMetaAggBucketAggregateTimeSec = &MetricMetaValue{ Kind: MetricKindValue, Description: `Time between agent bucket is received and fully aggregated into aggregator bucket. Set by aggregator. Max(value)@host shows agent responsible for longest aggregation.`, - MetricType: MetricSecond, + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -371,9 +429,13 @@ Set by aggregator. Max(value)@host shows agent responsible for longest aggregati const BuiltinMetricIDAggActiveSenders = -15 var BuiltinMetricMetaAggActiveSenders = &MetricMetaValue{ - Name: "__agg_active_senders", - Kind: MetricKindValue, - Description: "Number of insert lines between aggregator and clickhouse busy with insertion.", + Name: "__agg_active_senders", + Kind: MetricKindValue, + Description: "Number of insert lines between aggregator and clickhouse busy with insertion.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -389,10 +451,14 @@ var BuiltinMetricMetaAggActiveSenders = &MetricMetaValue{ const BuiltinMetricIDAggOutdatedAgents = -16 var BuiltinMetricMetaAggOutdatedAgents = &MetricMetaValue{ - Name: "__agg_outdated_agents", - Kind: MetricKindCounter, - Resolution: 60, - Description: "Number of outdated agents.", + Name: "__agg_outdated_agents", + Kind: MetricKindCounter, + Resolution: 60, + Description: "Number of outdated agents.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -412,9 +478,13 @@ var BuiltinMetricMetaAggOutdatedAgents = &MetricMetaValue{ const BuiltinMetricIDAgentDiskCacheErrors = -18 var BuiltinMetricMetaAgentDiskCacheErrors = &MetricMetaValue{ - Name: "__src_disc_cache_errors", - Kind: MetricKindCounter, - Description: "Disk cache errors. Written by agent.", + Name: "__src_disc_cache_errors", + Kind: MetricKindCounter, + Description: "Disk cache errors. Written by agent.", + NoSampleAgent: true, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "kind", ValueComments: convertToValueComments(map[int32]string{ @@ -434,6 +504,10 @@ var BuiltinMetricMetaTimingErrors = &MetricMetaValue{ Kind: MetricKindValue, Description: `Timing errors - sending data too early or too late. Set by either agent or aggregator, depending on status.`, + NoSampleAgent: true, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "status", ValueComments: convertToValueComments(map[int32]string{ @@ -460,10 +534,14 @@ Set by either agent or aggregator, depending on status.`, const BuiltinMetricIDAgentReceivedBatchSize = -21 var BuiltinMetricMetaAgentReceivedBatchSize = &MetricMetaValue{ - Name: "__src_ingested_metric_batch_size", - Kind: MetricKindValue, - Description: "Size in bytes of metric batches received by agent.\nCount is # of such batches.", - MetricType: MetricByte, + Name: "__src_ingested_metric_batch_size", + Kind: MetricKindValue, + Description: "Size in bytes of metric batches received by agent.\nCount is # of such batches.", + MetricType: MetricByte, + NoSampleAgent: true, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "format", ValueComments: convertToValueComments(packetFormatToValue), @@ -482,9 +560,13 @@ var BuiltinMetricMetaAgentReceivedBatchSize = &MetricMetaValue{ const BuiltinMetricIDAggMapping = -23 var BuiltinMetricMetaAggMapping = &MetricMetaValue{ - Name: "__agg_mapping_status", - Kind: MetricKindCounter, - Description: "Status of mapping on aggregator side.", + Name: "__agg_mapping_status", + Kind: MetricKindCounter, + Description: "Status of mapping on aggregator side.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -518,10 +600,14 @@ var BuiltinMetricMetaAggMapping = &MetricMetaValue{ const BuiltinMetricIDAggInsertTimeReal = -24 var BuiltinMetricMetaAggInsertTimeReal = &MetricMetaValue{ - Name: "__agg_insert_time_real", - Kind: MetricKindValue, - Description: "Time of aggregated bucket inserting into clickhouse took in this second.\nactual seconds inserted can be from the past.", - MetricType: MetricSecond, + Name: "__agg_insert_time_real", + Kind: MetricKindValue, + Description: "Time of aggregated bucket inserting into clickhouse took in this second.\nactual seconds inserted can be from the past.", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -555,10 +641,14 @@ var BuiltinMetricMetaAggInsertTimeReal = &MetricMetaValue{ const BuiltinMetricIDAgentHistoricQueueSize = -25 var BuiltinMetricMetaAgentHistoricQueueSize = &MetricMetaValue{ - Name: "__src_historic_queue_size_bytes", - Kind: MetricKindValue, - Description: "Historic queue size in memory and on disk.\nDisk size increases when second is written, decreases when file is deleted.", - MetricType: MetricByte, + Name: "__src_historic_queue_size_bytes", + Kind: MetricKindValue, + Description: "Historic queue size in memory and on disk.\nDisk size increases when second is written, decreases when file is deleted.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "storage", ValueComments: convertToValueComments(map[int32]string{ @@ -580,9 +670,13 @@ var BuiltinMetricMetaAgentHistoricQueueSize = &MetricMetaValue{ const BuiltinMetricIDAggHistoricSecondsWaiting = -26 var BuiltinMetricMetaAggHistoricSecondsWaiting = &MetricMetaValue{ - Name: "__agg_historic_seconds_waiting", - Kind: MetricKindValue, - Description: "Time difference of aggregated historic seconds waiting to be inserted via historic conveyor. Count is number of unique seconds waiting.", + Name: "__agg_historic_seconds_waiting", + Kind: MetricKindValue, + Description: "Time difference of aggregated historic seconds waiting to be inserted via historic conveyor. Count is number of unique seconds waiting.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -601,10 +695,14 @@ var BuiltinMetricMetaAggHistoricSecondsWaiting = &MetricMetaValue{ const BuiltinMetricIDAggInsertSizeReal = -27 var BuiltinMetricMetaAggInsertSizeReal = &MetricMetaValue{ - Name: "__agg_insert_size_real", - Kind: MetricKindValue, - Description: "Size of aggregated bucket inserted into clickhouse in this second (actual seconds inserted can be from the past).", - MetricType: MetricByte, + Name: "__agg_insert_size_real", + Kind: MetricKindValue, + Description: "Size of aggregated bucket inserted into clickhouse in this second (actual seconds inserted can be from the past).", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -638,10 +736,14 @@ var BuiltinMetricMetaAggInsertSizeReal = &MetricMetaValue{ const BuiltinMetricIDAgentMapping = -30 var BuiltinMetricMetaAgentMapping = &MetricMetaValue{ - Name: "__src_mapping_time", - Kind: MetricKindValue, - Description: "Time and status of mapping request.\nWritten by agent.", - MetricType: MetricSecond, + Name: "__src_mapping_time", + Kind: MetricKindValue, + Description: "Time and status of mapping request.\nWritten by agent.", + MetricType: MetricSecond, + NoSampleAgent: true, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "mapper", ValueComments: convertToValueComments(map[int32]string{ @@ -664,10 +766,14 @@ var BuiltinMetricMetaAgentMapping = &MetricMetaValue{ const BuiltinMetricIDAgentReceivedPacketSize = -31 var BuiltinMetricMetaAgentReceivedPacketSize = &MetricMetaValue{ - Name: "__src_ingested_packet_size", - Kind: MetricKindValue, - Description: "Size in bytes of packets received by agent. Also count is # of received packets.", - MetricType: MetricByte, + Name: "__src_ingested_packet_size", + Kind: MetricKindValue, + Description: "Size in bytes of packets received by agent. Also count is # of received packets.", + MetricType: MetricByte, + NoSampleAgent: true, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "format", ValueComments: convertToValueComments(packetFormatToValue), @@ -695,7 +801,11 @@ var BuiltinMetricMetaAggMappingCreated = &MetricMetaValue{ Description: `Status of mapping string tags to integer values. Value is actual integer value created (by incrementing global counter). Set by aggregator.`, - StringTopDescription: "Tag Values", + StringTopDescription: "Tag Values", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -731,11 +841,15 @@ Set by aggregator.`, const BuiltinMetricIDVersions = -34 var BuiltinMetricMetaVersions = &MetricMetaValue{ - Name: "__build_version", - Kind: MetricKindCounter, - Description: "Build Version (commit) of statshouse components.", - StringTopDescription: "Build Commit", - Resolution: 60, + Name: "__build_version", + Kind: MetricKindCounter, + Description: "Build Version (commit) of statshouse components.", + StringTopDescription: "Build Commit", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -757,10 +871,14 @@ var BuiltinMetricMetaVersions = &MetricMetaValue{ const BuiltinMetricIDBadges = -35 // copy of some other metrics for efficient show of errors and warnings var BuiltinMetricMetaBadges = &MetricMetaValue{ - Name: "__badges", - Kind: MetricKindValue, - Description: "System metric used to display UI badges above plot. Stores stripped copy of some other builtin metrics.", - Resolution: 5, + Name: "__badges", + Kind: MetricKindValue, + Description: "System metric used to display UI badges above plot. Stores stripped copy of some other builtin metrics.", + Resolution: 5, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, // this metric must be extremely lightweight + WithAggregatorID: false, // this metric must be extremely lightweight Tags: []MetricMetaTag{{ Description: "badge", ValueComments: convertToValueComments(map[int32]string{ @@ -789,6 +907,10 @@ var BuiltinMetricMetaAutoConfig = &MetricMetaValue{ Description: `Status of agent getConfig RPC message, used to configure sharding on agents. Set by aggregator, max host shows actual host of agent who connected. Ingress proxies first proxy request (to record host and IP of agent), then replace response with their own addresses.'`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -814,11 +936,15 @@ Ingress proxies first proxy request (to record host and IP of agent), then repla const BuiltinMetricIDJournalVersions = -37 // has smart custom sending logic var BuiltinMetricMetaJournalVersions = &MetricMetaValue{ - Name: "__metric_journal_version", - Kind: MetricKindCounter, - Description: "Metadata journal version plus stable hash of journal state.", - StringTopDescription: "Journal Hash", - Resolution: 60, + Name: "__metric_journal_version", + Kind: MetricKindCounter, + Description: "Metadata journal version plus stable hash of journal state.", + StringTopDescription: "Journal Hash", + Resolution: 60, + NoSampleAgent: true, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -840,10 +966,14 @@ var BuiltinMetricMetaJournalVersions = &MetricMetaValue{ const BuiltinMetricIDPromScrapeTime = -38 var BuiltinMetricMetaPromScrapeTime = &MetricMetaValue{ - Name: "__prom_scrape_time", - Kind: MetricKindValue, - Description: "Time of scraping prom metrics", - MetricType: MetricSecond, + Name: "__prom_scrape_time", + Kind: MetricKindValue, + Description: "Time of scraping prom metrics", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{ { Description: "-", @@ -868,11 +998,15 @@ const BuiltinMetricIDAgentHeartbeatArgs = -42 // TODO - remove, this metric w const BuiltinMetricIDUsageMemory = -43 var BuiltinMetricMetaUsageMemory = &MetricMetaValue{ - Name: "__usage_mem", - Kind: MetricKindValue, - Description: "Memory usage of statshouse components.", - MetricType: MetricByte, - Resolution: 60, + Name: "__usage_mem", + Kind: MetricKindValue, + Description: "Memory usage of statshouse components.", + MetricType: MetricByte, + Resolution: 60, + NoSampleAgent: true, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -882,11 +1016,15 @@ var BuiltinMetricMetaUsageMemory = &MetricMetaValue{ const BuiltinMetricIDUsageCPU = -44 var BuiltinMetricMetaUsageCPU = &MetricMetaValue{ - Name: "__usage_cpu", - Kind: MetricKindValue, - Description: "CPU usage of statshouse components, CPU seconds per second.", - MetricType: MetricSecond, - Resolution: 60, + Name: "__usage_cpu", + Kind: MetricKindValue, + Description: "CPU usage of statshouse components, CPU seconds per second.", + MetricType: MetricSecond, + Resolution: 60, + NoSampleAgent: true, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -901,30 +1039,42 @@ var BuiltinMetricMetaUsageCPU = &MetricMetaValue{ const BuiltinMetricIDGeneratorConstCounter = -45 var BuiltinMetricMetaGeneratorConstCounter = &MetricMetaValue{ - Name: "__fn_const_counter", - Kind: MetricKindCounter, - Description: "Counter generated on the fly by constant function", - Tags: []MetricMetaTag{}, + Name: "__fn_const_counter", + Kind: MetricKindCounter, + Description: "Counter generated on the fly by constant function", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, + Tags: []MetricMetaTag{}, } const BuiltinMetricIDGeneratorSinCounter = -46 var BuiltinMetricMetaGeneratorSinCounter = &MetricMetaValue{ - Name: "__fn_sin_counter", - Kind: MetricKindCounter, - Description: "Test counter generated on the fly by sine function", - Tags: []MetricMetaTag{}, + Name: "__fn_sin_counter", + Kind: MetricKindCounter, + Description: "Test counter generated on the fly by sine function", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, + Tags: []MetricMetaTag{}, } const BuiltinMetricIDHeartbeatVersion = -47 var BuiltinMetricMetaHeartbeatVersion = &MetricMetaValue{ - Name: "__heartbeat_version", - Kind: MetricKindValue, - Description: "Heartbeat value is uptime", - MetricType: MetricSecond, - StringTopDescription: "Build Commit", - Resolution: 60, + Name: "__heartbeat_version", + Kind: MetricKindValue, + Description: "Heartbeat value is uptime", + MetricType: MetricSecond, + StringTopDescription: "Build Commit", + Resolution: 60, + NoSampleAgent: true, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -955,12 +1105,16 @@ var BuiltinMetricMetaHeartbeatVersion = &MetricMetaValue{ const BuiltinMetricIDHeartbeatArgs = -48 // this metric was writing larger than allowed strings to DB in the past var BuiltinMetricMetaHeartbeatArgs = &MetricMetaValue{ - Name: "__heartbeat_args", - Kind: MetricKindValue, - Description: "Commandline of statshouse components.\nHeartbeat value is uptime.", - MetricType: MetricSecond, - StringTopDescription: "Arguments", - Resolution: 60, + Name: "__heartbeat_args", + Kind: MetricKindValue, + Description: "Commandline of statshouse components.\nHeartbeat value is uptime.", + MetricType: MetricSecond, + StringTopDescription: "Arguments", + Resolution: 60, + NoSampleAgent: true, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -996,9 +1150,13 @@ var BuiltinMetricMetaHeartbeatArgs = &MetricMetaValue{ const BuiltinMetricIDAPIBRS = -50 var BuiltinMetricMetaAPIBRS = &MetricMetaValue{ // TODO - harmonize - Name: "__api_big_response_storage_size", - Kind: MetricKindValue, - Description: "Size of storage inside API of big response chunks", + Name: "__api_big_response_storage_size", + Kind: MetricKindValue, + Description: "Size of storage inside API of big response chunks", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -1008,26 +1166,38 @@ var BuiltinMetricMetaAPIBRS = &MetricMetaValue{ // TODO - harmonize // BuiltinMetricIDAPIEndpointServiceTime = -52 // deprecated, replaced by "__api_service_time" const BuiltinMetricIDBudgetHost = -53 // these 2 metrics are invisible, but host mapping is flood-protected by their names var BuiltinMetricMetaBudgetHost = &MetricMetaValue{ - Name: "__budget_host", - Kind: MetricKindCounter, - Description: "Invisible metric used only for accounting budget to create host mappings", - Tags: []MetricMetaTag{}, + Name: "__budget_host", + Kind: MetricKindCounter, + Description: "Invisible metric used only for accounting budget to create host mappings", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, + Tags: []MetricMetaTag{}, } const BuiltinMetricIDBudgetAggregatorHost = -54 // we want to see limits properly credited in flood meta metric tags var BuiltinMetricMetaBudgetAggregatorHost = &MetricMetaValue{ - Name: "__budget_aggregator_host", - Kind: MetricKindCounter, - Description: "Invisible metric used only for accounting budget to create host mappings of aggregators themselves", - Tags: []MetricMetaTag{}, + Name: "__budget_aggregator_host", + Kind: MetricKindCounter, + Description: "Invisible metric used only for accounting budget to create host mappings of aggregators themselves", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, + Tags: []MetricMetaTag{}, } const BuiltinMetricIDAPIActiveQueries = -55 var BuiltinMetricMetaAPIActiveQueries = &MetricMetaValue{ - Name: "__api_active_queries", - Kind: MetricKindValue, - Description: "Active queries to clickhouse by API.\nRequests are assigned to lanes by estimated processing time.", + Name: "__api_active_queries", + Kind: MetricKindValue, + Description: "Active queries to clickhouse by API.\nRequests are assigned to lanes by estimated processing time.", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "-", // if we need another component }, { @@ -1050,10 +1220,14 @@ var BuiltinMetricMetaAPIActiveQueries = &MetricMetaValue{ const BuiltinMetricIDRPCRequests = -56 var BuiltinMetricMetaRPCRequests = &MetricMetaValue{ - Name: "__rpc_request_size", - Kind: MetricKindValue, - Description: "Size of RPC request bodies.\nFor ingress proxy, key_id can be used to identify senders.", - MetricType: MetricByte, + Name: "__rpc_request_size", + Kind: MetricKindValue, + Description: "Size of RPC request bodies.\nFor ingress proxy, key_id can be used to identify senders.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -1099,10 +1273,14 @@ var BuiltinMetricMetaRPCRequests = &MetricMetaValue{ const BuiltinMetricIDBudgetUnknownMetric = -57 var BuiltinMetricMetaBudgetUnknownMetric = &MetricMetaValue{ - Name: "__budget_unknown_metric", - Kind: MetricKindCounter, - Description: "Invisible metric used only for accounting budget to create mappings with metric not found", - Tags: []MetricMetaTag{}, + Name: "__budget_unknown_metric", + Kind: MetricKindCounter, + Description: "Invisible metric used only for accounting budget to create mappings with metric not found", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, + Tags: []MetricMetaTag{}, } // BuiltinMetricIDHeartbeatArgs2 = -58 // not recorded any more @@ -1117,6 +1295,10 @@ var BuiltinMetricMetaContributorsLog = &MetricMetaValue{ Timestamps of all inserted seconds per second are recorded here in key1. Value is delta between second value and time it was inserted. To see which seconds change when, use __contributors_log_rev`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "timestamp", RawKind: "timestamp", @@ -1131,6 +1313,10 @@ var BuiltinMetricMetaContributorsLogRev = &MetricMetaValue{ Description: `Reverse index of __contributors_log, used to invalidate API caches. key1 is UNIX timestamp of second when this second was changed. Value is delta between second value and time it was inserted.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "insert_timestamp", RawKind: "timestamp", @@ -1140,18 +1326,26 @@ Value is delta between second value and time it was inserted.`, const BuiltinMetricIDGeneratorGapsCounter = -63 var BuiltinMetricMetaGeneratorGapsCounter = &MetricMetaValue{ - Name: "__fn_gaps_counter", - Kind: MetricKindCounter, - Description: "Test counter with constant value, but with multiple gaps", - Tags: []MetricMetaTag{}, + Name: "__fn_gaps_counter", + Kind: MetricKindCounter, + Description: "Test counter with constant value, but with multiple gaps", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, + Tags: []MetricMetaTag{}, } const BuiltinMetricIDGroupSizeBeforeSampling = -64 var BuiltinMetricMetaGroupSizeBeforeSampling = &MetricMetaValue{ - Name: "__group_size_before_sampling", - Kind: MetricKindValue, - Description: "Group size before sampling, bytes.", + Name: "__group_size_before_sampling", + Kind: MetricKindValue, + Description: "Group size before sampling, bytes.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -1170,10 +1364,14 @@ var BuiltinMetricMetaGroupSizeBeforeSampling = &MetricMetaValue{ const BuiltinMetricIDGroupSizeAfterSampling = -65 var BuiltinMetricMetaGroupSizeAfterSampling = &MetricMetaValue{ - Name: "__group_size_after_sampling", - Kind: MetricKindValue, - Description: "Group size after sampling, bytes.", - MetricType: MetricByte, + Name: "__group_size_after_sampling", + Kind: MetricKindValue, + Description: "Group size after sampling, bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -1195,8 +1393,12 @@ var BuiltinMetricMetaAPISelectBytes = &MetricMetaValue{ Name: "__api_ch_select_bytes", Kind: MetricKindValue, // TODO replace with logs - StringTopDescription: "error", - Description: "Number of bytes was handled by ClickHouse SELECT query", + StringTopDescription: "error", + Description: "Number of bytes was handled by ClickHouse SELECT query", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "query type", }}, @@ -1208,8 +1410,12 @@ var BuiltinMetricMetaAPISelectRows = &MetricMetaValue{ Name: "__api_ch_select_rows", Kind: MetricKindValue, // TODO replace with logs - StringTopDescription: "error", - Description: "Number of rows was handled by ClickHouse SELECT query", + StringTopDescription: "error", + Description: "Number of rows was handled by ClickHouse SELECT query", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "query type", }}, @@ -1218,10 +1424,14 @@ var BuiltinMetricMetaAPISelectRows = &MetricMetaValue{ const BuiltinMetricIDAPISelectDuration = -68 var BuiltinMetricMetaAPISelectDuration = &MetricMetaValue{ - Name: "__api_ch_select_duration", - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "Duration of clickhouse query", + Name: "__api_ch_select_duration", + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "Duration of clickhouse query", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{ { Description: "query type", @@ -1255,10 +1465,14 @@ var BuiltinMetricMetaAPISelectDuration = &MetricMetaValue{ const BuiltinMetricIDAgentHistoricQueueSizeSum = -69 var BuiltinMetricMetaAgentHistoricQueueSizeSum = &MetricMetaValue{ - Name: "__src_historic_queue_size_sum_bytes", - Kind: MetricKindValue, - Description: "Historic queue size in memory and on disk, sum for shards sent to every shard.\nCan be compared with __src_historic_queue_size_bytes to find if subset of aggregators is inaccessible.", - MetricType: MetricByte, + Name: "__src_historic_queue_size_sum_bytes", + Kind: MetricKindValue, + Description: "Historic queue size in memory and on disk, sum for shards sent to every shard.\nCan be compared with __src_historic_queue_size_bytes to find if subset of aggregators is inaccessible.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "storage", ValueComments: convertToValueComments(map[int32]string{ @@ -1280,9 +1494,13 @@ var BuiltinMetricMetaAgentHistoricQueueSizeSum = &MetricMetaValue{ const BuiltinMetricIDAPISourceSelectRows = -70 var BuiltinMetricMetaAPISourceSelectRows = &MetricMetaValue{ - Name: "__api_ch_source_select_rows", - Kind: MetricKindValue, - Description: "Value of this metric number of rows was selected from DB or cache", + Name: "__api_ch_source_select_rows", + Kind: MetricKindValue, + Description: "Value of this metric number of rows was selected from DB or cache", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{ { Description: "source type", @@ -1303,10 +1521,14 @@ var BuiltinMetricMetaAPISourceSelectRows = &MetricMetaValue{ const BuiltinMetricIDSystemMetricScrapeDuration = -71 var BuiltinMetricMetaSystemMetricScrapeDuration = &MetricMetaValue{ - Name: "__system_metrics_duration", - Kind: MetricKindValue, - Description: "System metrics scrape duration in seconds", - MetricType: MetricSecond, + Name: "__system_metrics_duration", + Kind: MetricKindValue, + Description: "System metrics scrape duration in seconds", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "collector", ValueComments: convertToValueComments(map[int32]string{ @@ -1328,10 +1550,14 @@ var BuiltinMetricMetaSystemMetricScrapeDuration = &MetricMetaValue{ const BuiltinMetricIDMetaServiceTime = -72 var BuiltinMetricMetaMetaServiceTime = &MetricMetaValue{ // TODO - harmonize - Name: "__meta_rpc_service_time", - Kind: MetricKindValue, - Description: "Time to handle RPC query by meta.", - MetricType: MetricSecond, + Name: "__meta_rpc_service_time", + Kind: MetricKindValue, + Description: "Time to handle RPC query by meta.", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -1346,9 +1572,13 @@ var BuiltinMetricMetaMetaServiceTime = &MetricMetaValue{ // TODO - harmonize const BuiltinMetricIDMetaClientWaits = -73 var BuiltinMetricMetaMetaClientWaits = &MetricMetaValue{ // TODO - harmonize - Name: "__meta_load_journal_client_waits", - Kind: MetricKindValue, - Description: "Number of clients waiting journal updates", + Name: "__meta_load_journal_client_waits", + Kind: MetricKindValue, + Description: "Number of clients waiting journal updates", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -1357,20 +1587,28 @@ var BuiltinMetricMetaMetaClientWaits = &MetricMetaValue{ // TODO - harmonize const BuiltinMetricIDAgentUDPReceiveBufferSize = -74 var BuiltinMetricMetaAgentUDPReceiveBufferSize = &MetricMetaValue{ - Name: "__src_udp_receive_buffer_size", - Kind: MetricKindValue, - Resolution: 60, - Description: "Size in bytes of agent UDP receive buffer.", - MetricType: MetricByte, + Name: "__src_udp_receive_buffer_size", + Kind: MetricKindValue, + Resolution: 60, + Description: "Size in bytes of agent UDP receive buffer.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, } const BuiltinMetricIDAPIMetricUsage = -75 var BuiltinMetricMetaAPIMetricUsage = &MetricMetaValue{ - Name: "__api_metric_usage", - Resolution: 60, - Kind: MetricKindCounter, - Description: "Metric usage", + Name: "__api_metric_usage", + Resolution: 60, + Kind: MetricKindCounter, + Description: "Metric usage", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{ { Description: "type", @@ -1392,10 +1630,14 @@ var BuiltinMetricMetaAPIMetricUsage = &MetricMetaValue{ const BuiltinMetricIDAPIServiceTime = -76 var BuiltinMetricMetaAPIServiceTime = &MetricMetaValue{ - Name: "__api_service_time", - Kind: MetricKindValue, - Description: "Time to handle API query.", - MetricType: MetricSecond, + Name: "__api_service_time", + Kind: MetricKindValue, + Description: "Time to handle API query.", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "endpoint", }, { @@ -1438,10 +1680,14 @@ var BuiltinMetricMetaAPIServiceTime = &MetricMetaValue{ const BuiltinMetricIDAPIResponseTime = -77 var BuiltinMetricMetaAPIResponseTime = &MetricMetaValue{ - Name: "__api_response_time", - Kind: MetricKindValue, - Description: "Time to handle and respond to query by API", - MetricType: MetricSecond, + Name: "__api_response_time", + Kind: MetricKindValue, + Description: "Time to handle and respond to query by API", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "endpoint", }, { @@ -1481,11 +1727,15 @@ var BuiltinMetricMetaAPIResponseTime = &MetricMetaValue{ const BuiltinMetricIDSrcTestConnection = -78 var BuiltinMetricMetaSrcTestConnection = &MetricMetaValue{ - Name: "__src_test_connection", - Kind: MetricKindValue, - Resolution: 60, - Description: "Duration of call test connection rpc method", - MetricType: MetricSecond, + Name: "__src_test_connection", + Kind: MetricKindValue, + Resolution: 60, + Description: "Duration of call test connection rpc method", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -1504,11 +1754,15 @@ var BuiltinMetricMetaSrcTestConnection = &MetricMetaValue{ const BuiltinMetricIDAgentAggregatorTimeDiff = -79 var BuiltinMetricMetaAgentAggregatorTimeDiff = &MetricMetaValue{ - Name: "__src_agg_time_diff", - Kind: MetricKindValue, - Resolution: 60, - Description: "Aggregator time - agent time when start testConnection", - MetricType: MetricSecond, + Name: "__src_agg_time_diff", + Kind: MetricKindValue, + Resolution: 60, + Description: "Aggregator time - agent time when start testConnection", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -1518,9 +1772,13 @@ var BuiltinMetricMetaAgentAggregatorTimeDiff = &MetricMetaValue{ const BuiltinMetricIDSrcSamplingMetricCount = -80 var BuiltinMetricMetaSrcSamplingMetricCount = &MetricMetaValue{ - Name: "__src_sampling_metric_count", - Kind: MetricKindValue, - Description: `Metric count processed by sampler on agent.`, + Name: "__src_sampling_metric_count", + Kind: MetricKindValue, + Description: `Metric count processed by sampler on agent.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Name: "component", ValueComments: convertToValueComments(componentToValue), @@ -1530,9 +1788,13 @@ var BuiltinMetricMetaSrcSamplingMetricCount = &MetricMetaValue{ const BuiltinMetricIDAggSamplingMetricCount = -81 var BuiltinMetricMetaAggSamplingMetricCount = &MetricMetaValue{ - Name: "__agg_sampling_metric_count", - Kind: MetricKindValue, - Description: `Metric count processed by sampler on aggregator.`, + Name: "__agg_sampling_metric_count", + Kind: MetricKindValue, + Description: `Metric count processed by sampler on aggregator.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Name: "conveyor", ValueComments: convertToValueComments(conveyorToValue), @@ -1542,10 +1804,14 @@ var BuiltinMetricMetaAggSamplingMetricCount = &MetricMetaValue{ const BuiltinMetricIDSrcSamplingSizeBytes = -82 var BuiltinMetricMetaSrcSamplingSizeBytes = &MetricMetaValue{ - Name: "__src_sampling_size_bytes", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: `Size in bytes processed by sampler on agent.`, + Name: "__src_sampling_size_bytes", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: `Size in bytes processed by sampler on agent.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Name: "component", ValueComments: convertToValueComments(componentToValue), @@ -1577,10 +1843,14 @@ var BuiltinMetricMetaSrcSamplingSizeBytes = &MetricMetaValue{ const BuiltinMetricIDAggSamplingSizeBytes = -83 var BuiltinMetricMetaAggSamplingSizeBytes = &MetricMetaValue{ - Name: "__agg_sampling_size_bytes", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: `Size in bytes processed by sampler on aggregator.`, + Name: "__agg_sampling_size_bytes", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: `Size in bytes processed by sampler on aggregator.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Name: "conveyor", ValueComments: convertToValueComments(conveyorToValue), @@ -1612,20 +1882,28 @@ var BuiltinMetricMetaAggSamplingSizeBytes = &MetricMetaValue{ const BuiltinMetricIDUIErrors = -84 var BuiltinMetricMetaUIErrors = &MetricMetaValue{ - Name: "__ui_errors", - Kind: MetricKindValue, - Description: `Errors on the frontend.`, - StringTopDescription: "error_string", - Tags: []MetricMetaTag{{Description: "environment"}}, + Name: "__ui_errors", + Kind: MetricKindValue, + Description: `Errors on the frontend.`, + StringTopDescription: "error_string", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, + Tags: []MetricMetaTag{{Description: "environment"}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}}, } const BuiltinMetricIDStatsHouseErrors = -85 var BuiltinMetricMetaStatsHouseErrors = &MetricMetaValue{ - Name: "__statshouse_errors", - Kind: MetricKindCounter, - Description: `Always empty metric because SH don't have errors'`, - StringTopDescription: "error_string", + Name: "__statshouse_errors", + Kind: MetricKindCounter, + Description: `Always empty metric because SH don't have errors'`, + StringTopDescription: "error_string", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{ { Description: "error_type", @@ -1634,16 +1912,20 @@ var BuiltinMetricMetaStatsHouseErrors = &MetricMetaValue{ TagValueIDDMESGParseError: "dmesg_parse", TagValueIDAPIPanicError: "api_panic", }), - }}, + }, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}}, } const BuiltinMetricIDSrcSamplingBudget = -86 var BuiltinMetricMetaSrcSamplingBudget = &MetricMetaValue{ - Name: "__src_sampling_budget", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: `Budget allocated on agent.`, + Name: "__src_sampling_budget", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: `Budget allocated on agent.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Name: "component", ValueComments: convertToValueComments(componentToValue), @@ -1653,10 +1935,14 @@ var BuiltinMetricMetaSrcSamplingBudget = &MetricMetaValue{ const BuiltinMetricIDAggSamplingBudget = -87 var BuiltinMetricMetaAggSamplingBudget = &MetricMetaValue{ - Name: "__agg_sampling_budget", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: `Budget allocated on aggregator.`, + Name: "__agg_sampling_budget", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: `Budget allocated on aggregator.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Name: "conveyor", ValueComments: convertToValueComments(conveyorToValue), @@ -1666,10 +1952,14 @@ var BuiltinMetricMetaAggSamplingBudget = &MetricMetaValue{ const BuiltinMetricIDSrcSamplingGroupBudget = -88 var BuiltinMetricMetaSrcSamplingGroupBudget = &MetricMetaValue{ - Name: "__src_sampling_group_budget", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: `Group budget allocated on agent.`, + Name: "__src_sampling_group_budget", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: `Group budget allocated on agent.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Name: "component", ValueComments: convertToValueComments(componentToValue), @@ -1695,10 +1985,14 @@ var BuiltinMetricMetaSrcSamplingGroupBudget = &MetricMetaValue{ const BuiltinMetricIDAggSamplingGroupBudget = -89 var BuiltinMetricMetaAggSamplingGroupBudget = &MetricMetaValue{ - Name: "__agg_sampling_group_budget", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: `Group budget allocated on aggregator.`, + Name: "__agg_sampling_group_budget", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: `Group budget allocated on aggregator.`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Name: "conveyor", ValueComments: convertToValueComments(conveyorToValue), @@ -1724,10 +2018,14 @@ var BuiltinMetricMetaAggSamplingGroupBudget = &MetricMetaValue{ const BuiltinMetricIDPromQLEngineTime = -90 var BuiltinMetricMetaPromQLEngineTime = &MetricMetaValue{ - Name: "__promql_engine_time", - Kind: MetricKindValue, - Description: "Time spent in PromQL engine", - MetricType: MetricSecond, + Name: "__promql_engine_time", + Kind: MetricKindValue, + Description: "Time spent in PromQL engine", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Name: "host", Description: "API host", @@ -1783,9 +2081,13 @@ var BuiltinMetricMetaPromQLEngineTime = &MetricMetaValue{ const BuiltinMetricIDAPICacheHit = -91 var BuiltinMetricMetaAPICacheHit = &MetricMetaValue{ - Name: "__api_cache_hit_rate", - Kind: MetricKindValue, - Description: `API cache hit rate`, + Name: "__api_cache_hit_rate", + Kind: MetricKindValue, + Description: `API cache hit rate`, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "source", }, { @@ -1802,10 +2104,14 @@ var BuiltinMetricMetaAPICacheHit = &MetricMetaValue{ const BuiltinMetricIDAggScrapeTargetDispatch = -92 var BuiltinMetricMetaAggScrapeTargetDispatch = &MetricMetaValue{ - Name: "__agg_scrape_target_dispatch", - Kind: MetricKindCounter, - Description: "Scrape target-to-agent assigment events", - StringTopDescription: "agent_host", + Name: "__agg_scrape_target_dispatch", + Kind: MetricKindCounter, + Description: "Scrape target-to-agent assigment events", + StringTopDescription: "agent_host", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{ { Description: "status", @@ -1829,18 +2135,26 @@ var BuiltinMetricMetaAggScrapeTargetDispatch = &MetricMetaValue{ const BuiltinMetricIDAggScrapeTargetDiscovery = -93 var BuiltinMetricMetaAggScrapeTargetDiscovery = &MetricMetaValue{ - Name: "__agg_scrape_target_discovery", - Kind: MetricKindCounter, - Description: "Scrape targets found by service discovery", - StringTopDescription: "scrape_target", + Name: "__agg_scrape_target_discovery", + Kind: MetricKindCounter, + Description: "Scrape targets found by service discovery", + StringTopDescription: "scrape_target", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, } const BuiltinMetricIDAggScrapeConfigHash = -94 var BuiltinMetricMetaAggScrapeConfigHash = &MetricMetaValue{ - Name: "__agg_scrape_config_hash", - Kind: MetricKindCounter, - Description: "Scrape configuration string SHA1 hash", + Name: "__agg_scrape_config_hash", + Kind: MetricKindCounter, + Description: "Scrape configuration string SHA1 hash", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{ { Description: "config_hash", @@ -1853,10 +2167,14 @@ var BuiltinMetricMetaAggScrapeConfigHash = &MetricMetaValue{ const BuiltinMetricIDAggSamplingTime = -95 var BuiltinMetricMetaAggSamplingTime = &MetricMetaValue{ - Name: "__agg_sampling_time", - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "Time sampling this second took. Written when second is inserted, which can be much later.", + Name: "__agg_sampling_time", + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "Time sampling this second took. Written when second is inserted, which can be much later.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -1872,10 +2190,14 @@ var BuiltinMetricMetaAggSamplingTime = &MetricMetaValue{ const BuiltinMetricIDAgentDiskCacheSize = -96 var BuiltinMetricMetaAgentDiskCacheSize = &MetricMetaValue{ - Name: "__src_disk_cache_size", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "Size of agent mapping cache", + Name: "__src_disk_cache_size", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "Size of agent mapping cache", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -1888,9 +2210,13 @@ var BuiltinMetricMetaAgentDiskCacheSize = &MetricMetaValue{ const BuiltinMetricIDAggContributors = -97 var BuiltinMetricMetaAggContributors = &MetricMetaValue{ - Name: "__agg_contributors", - Kind: MetricKindValue, - Description: "Number of contributors used to calculate sampling budget.", + Name: "__agg_contributors", + Kind: MetricKindValue, + Description: "Number of contributors used to calculate sampling budget.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -1904,10 +2230,14 @@ var BuiltinMetricMetaAggContributors = &MetricMetaValue{ const BuiltinMetricIDAPICacheBytesAlloc = -99 var BuiltinMetricMetaAPICacheBytesAlloc = &MetricMetaValue{ - Name: "__api_cache_bytes_alloc", - Kind: MetricKindValue, - Description: "API cache memory allocation in bytes.", - MetricType: MetricByte, + Name: "__api_cache_bytes_alloc", + Kind: MetricKindValue, + Description: "API cache memory allocation in bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -1922,10 +2252,14 @@ var BuiltinMetricMetaAPICacheBytesAlloc = &MetricMetaValue{ const BuiltinMetricIDAPICacheBytesFree = -100 var BuiltinMetricMetaAPICacheBytesFree = &MetricMetaValue{ - Name: "__api_cache_bytes_free", - Kind: MetricKindValue, - Description: "API cache memory deallocation in bytes.", - MetricType: MetricByte, + Name: "__api_cache_bytes_free", + Kind: MetricKindValue, + Description: "API cache memory deallocation in bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -1943,11 +2277,15 @@ var BuiltinMetricMetaAPICacheBytesFree = &MetricMetaValue{ const BuiltinMetricIDAPICacheBytesTotal = -101 var BuiltinMetricMetaAPICacheBytesTotal = &MetricMetaValue{ - Name: "__api_cache_bytes_total", - Kind: MetricKindValue, - Resolution: 15, - Description: "API cache size in bytes.", - MetricType: MetricByte, + Name: "__api_cache_bytes_total", + Kind: MetricKindValue, + Resolution: 15, + Description: "API cache size in bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -1962,10 +2300,14 @@ var BuiltinMetricMetaAPICacheBytesTotal = &MetricMetaValue{ const BuiltinMetricIDAPICacheAgeEvict = -102 var BuiltinMetricMetaAPICacheAgeEvict = &MetricMetaValue{ - Name: "__api_cache_age_evict", - Kind: MetricKindValue, - Description: "API cache entry age when evicted in seconds.", - MetricType: MetricSecond, + Name: "__api_cache_age_evict", + Kind: MetricKindValue, + Description: "API cache entry age when evicted in seconds.", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -1983,11 +2325,15 @@ var BuiltinMetricMetaAPICacheAgeEvict = &MetricMetaValue{ const BuiltinMetricIDAPICacheAgeTotal = -103 var BuiltinMetricMetaAPICacheAgeTotal = &MetricMetaValue{ - Name: "__api_cache_age_total", - Kind: MetricKindValue, - Resolution: 15, - Description: "API cache age in seconds.", - MetricType: MetricSecond, + Name: "__api_cache_age_total", + Kind: MetricKindValue, + Resolution: 15, + Description: "API cache age in seconds.", + MetricType: MetricSecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -2002,10 +2348,14 @@ var BuiltinMetricMetaAPICacheAgeTotal = &MetricMetaValue{ const BuiltinMetricIDAPIBufferBytesAlloc = -104 var BuiltinMetricMetaAPIBufferBytesAlloc = &MetricMetaValue{ - Name: "__api_buffer_bytes_alloc", - Kind: MetricKindValue, - Description: "API buffer allocation in bytes.", - MetricType: MetricByte, + Name: "__api_buffer_bytes_alloc", + Kind: MetricKindValue, + Description: "API buffer allocation in bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -2017,10 +2367,14 @@ var BuiltinMetricMetaAPIBufferBytesAlloc = &MetricMetaValue{ const BuiltinMetricIDAPIBufferBytesFree = -105 var BuiltinMetricMetaAPIBufferBytesFree = &MetricMetaValue{ - Name: "__api_buffer_bytes_free", - Kind: MetricKindValue, - Description: "API buffer deallocation in bytes.", - MetricType: MetricByte, + Name: "__api_buffer_bytes_free", + Kind: MetricKindValue, + Description: "API buffer deallocation in bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -2032,10 +2386,14 @@ var BuiltinMetricMetaAPIBufferBytesFree = &MetricMetaValue{ const BuiltinMetricIDAPIBufferBytesTotal = -106 var BuiltinMetricMetaAPIBufferBytesTotal = &MetricMetaValue{ - Name: "__api_buffer_bytes_total", - Kind: MetricKindValue, - Description: "API buffer pool size in bytes.", - MetricType: MetricByte, + Name: "__api_buffer_bytes_total", + Kind: MetricKindValue, + Description: "API buffer pool size in bytes.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2044,10 +2402,14 @@ var BuiltinMetricMetaAPIBufferBytesTotal = &MetricMetaValue{ const BuiltinMetricIDAutoCreateMetric = -107 var BuiltinMetricMetaAutoCreateMetric = &MetricMetaValue{ - Name: "__agg_autocreate_metric", - Kind: MetricKindCounter, - Description: "Event of automatically created metrics.", - MetricType: MetricByte, + Name: "__agg_autocreate_metric", + Kind: MetricKindCounter, + Description: "Event of automatically created metrics.", + MetricType: MetricByte, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "action", ValueComments: map[string]string{ @@ -2066,10 +2428,14 @@ var BuiltinMetricMetaAutoCreateMetric = &MetricMetaValue{ const BuiltinMetricIDRestartTimings = -108 var BuiltinMetricMetaRestartTimings = &MetricMetaValue{ - Name: "__src_restart_timings", - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "Time of various restart phases (inactive is time between process stop and start)", + Name: "__src_restart_timings", + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "Time of various restart phases (inactive is time between process stop and start)", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "component", ValueComments: convertToValueComments(componentToValue), @@ -2100,10 +2466,14 @@ var BuiltinMetricMetaRestartTimings = &MetricMetaValue{ const BuiltinMetricIDGCDuration = -109 var BuiltinMetricMetaGCDuration = &MetricMetaValue{ - Name: "__gc_duration", - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "Count - number of GC, Value - time spent to gc", + Name: "__gc_duration", + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "Count - number of GC, Value - time spent to gc", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "-", // reserved for host }, @@ -2116,9 +2486,13 @@ var BuiltinMetricMetaGCDuration = &MetricMetaValue{ const BuiltinMetricIDAggHistoricHostsWaiting = -110 var BuiltinMetricMetaAggHistoricHostsWaiting = &MetricMetaValue{ - Name: "__agg_historic_hosts_waiting", - Kind: MetricKindValue, - Description: "Approximate number of different hosts waiting with historic data.", + Name: "__agg_historic_hosts_waiting", + Kind: MetricKindValue, + Description: "Approximate number of different hosts waiting with historic data.", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -2137,10 +2511,14 @@ var BuiltinMetricMetaAggHistoricHostsWaiting = &MetricMetaValue{ const BuiltinMetricIDAggSamplingEngineTime = -111 var BuiltinMetricMetaAggSamplingEngineTime = &MetricMetaValue{ - Name: "__agg_sampling_engine_time", - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "Time spent in sampling engine", + Name: "__agg_sampling_engine_time", + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "Time spent in sampling engine", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "phase", ValueComments: map[string]string{ @@ -2163,9 +2541,13 @@ var BuiltinMetricMetaAggSamplingEngineTime = &MetricMetaValue{ const BuiltinMetricIDAggSamplingEngineKeys = -112 var BuiltinMetricMetaAggSamplingEngineKeys = &MetricMetaValue{ - Name: "__agg_sampling_engine_keys", - Kind: MetricKindCounter, - Description: "Number of series went through sampling engine", + Name: "__agg_sampling_engine_keys", + Kind: MetricKindCounter, + Description: "Number of series went through sampling engine", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -2181,10 +2563,14 @@ var BuiltinMetricMetaAggSamplingEngineKeys = &MetricMetaValue{ const BuiltinMetricIDProxyAcceptHandshakeError = -113 var BuiltinMetricMetaProxyAcceptHandshakeError = &MetricMetaValue{ - Name: "__igp_accept_handshake_error", - Kind: MetricKindCounter, - Description: "Proxy refused to accept incoming connection because of failed handshake.", - StringTopDescription: "remote_ip", + Name: "__igp_accept_handshake_error", + Kind: MetricKindCounter, + Description: "Proxy refused to accept incoming connection because of failed handshake.", + StringTopDescription: "remote_ip", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }, { @@ -2197,10 +2583,14 @@ var BuiltinMetricMetaProxyAcceptHandshakeError = &MetricMetaValue{ const BuiltinMetricIDProxyVmSize = -114 var BuiltinMetricMetaProxyVmSize = &MetricMetaValue{ - Name: "__igp_vm_size", - Kind: MetricKindValue, - Description: "StatsHouse proxy virtual memory size.", - Resolution: 60, + Name: "__igp_vm_size", + Kind: MetricKindValue, + Description: "StatsHouse proxy virtual memory size.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2209,10 +2599,14 @@ var BuiltinMetricMetaProxyVmSize = &MetricMetaValue{ const BuiltinMetricIDProxyVmRSS = -115 var BuiltinMetricMetaProxyVmRSS = &MetricMetaValue{ - Name: "__igp_vm_rss", - Kind: MetricKindValue, - Description: "StatsHouse proxy resident set size.", - Resolution: 60, + Name: "__igp_vm_rss", + Kind: MetricKindValue, + Description: "StatsHouse proxy resident set size.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2221,10 +2615,14 @@ var BuiltinMetricMetaProxyVmRSS = &MetricMetaValue{ const BuiltinMetricIDProxyHeapAlloc = -116 var BuiltinMetricMetaProxyHeapAlloc = &MetricMetaValue{ - Name: "__igp_heap_alloc", - Kind: MetricKindValue, - Description: "StatsHouse proxy bytes of allocated heap objects.", - Resolution: 60, + Name: "__igp_heap_alloc", + Kind: MetricKindValue, + Description: "StatsHouse proxy bytes of allocated heap objects.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2233,10 +2631,14 @@ var BuiltinMetricMetaProxyHeapAlloc = &MetricMetaValue{ const BuiltinMetricIDProxyHeapSys = -117 var BuiltinMetricMetaProxyHeapSys = &MetricMetaValue{ - Name: "__igp_heap_sys", - Kind: MetricKindValue, - Description: "StatsHouse proxy bytes of heap memory obtained from the OS.", - Resolution: 60, + Name: "__igp_heap_sys", + Kind: MetricKindValue, + Description: "StatsHouse proxy bytes of heap memory obtained from the OS.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2245,10 +2647,14 @@ var BuiltinMetricMetaProxyHeapSys = &MetricMetaValue{ const BuiltinMetricIDProxyHeapIdle = -118 var BuiltinMetricMetaProxyHeapIdle = &MetricMetaValue{ - Name: "__igp_heap_idle", - Kind: MetricKindValue, - Description: "StatsHouse proxy bytes in idle (unused) spans.", - Resolution: 60, + Name: "__igp_heap_idle", + Kind: MetricKindValue, + Description: "StatsHouse proxy bytes in idle (unused) spans.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2257,10 +2663,14 @@ var BuiltinMetricMetaProxyHeapIdle = &MetricMetaValue{ const BuiltinMetricIDProxyHeapInuse = -119 var BuiltinMetricMetaProxyHeapInuse = &MetricMetaValue{ - Name: "__igp_heap_inuse", - Kind: MetricKindValue, - Description: "StatsHouse proxy bytes in in-use spans.", - Resolution: 60, + Name: "__igp_heap_inuse", + Kind: MetricKindValue, + Description: "StatsHouse proxy bytes in in-use spans.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2269,10 +2679,14 @@ var BuiltinMetricMetaProxyHeapInuse = &MetricMetaValue{ const BuiltinMetricIDApiVmSize = -120 var BuiltinMetricMetaApiVmSize = &MetricMetaValue{ - Name: "__api_vm_size", - Kind: MetricKindValue, - Description: "StatsHouse API virtual memory size.", - Resolution: 60, + Name: "__api_vm_size", + Kind: MetricKindValue, + Description: "StatsHouse API virtual memory size.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2281,10 +2695,14 @@ var BuiltinMetricMetaApiVmSize = &MetricMetaValue{ const BuiltinMetricIDApiVmRSS = -121 var BuiltinMetricMetaApiVmRSS = &MetricMetaValue{ - Name: "__api_vm_rss", - Kind: MetricKindValue, - Description: "StatsHouse API resident set size.", - Resolution: 60, + Name: "__api_vm_rss", + Kind: MetricKindValue, + Description: "StatsHouse API resident set size.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2293,10 +2711,14 @@ var BuiltinMetricMetaApiVmRSS = &MetricMetaValue{ const BuiltinMetricIDApiHeapAlloc = -122 var BuiltinMetricMetaApiHeapAlloc = &MetricMetaValue{ - Name: "__api_heap_alloc", - Kind: MetricKindValue, - Description: "StatsHouse API bytes of allocated heap objects.", - Resolution: 60, + Name: "__api_heap_alloc", + Kind: MetricKindValue, + Description: "StatsHouse API bytes of allocated heap objects.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2305,10 +2727,14 @@ var BuiltinMetricMetaApiHeapAlloc = &MetricMetaValue{ const BuiltinMetricIDApiHeapSys = -123 var BuiltinMetricMetaApiHeapSys = &MetricMetaValue{ - Name: "__api_heap_sys", - Kind: MetricKindValue, - Description: "StatsHouse API bytes of heap memory obtained from the OS.", - Resolution: 60, + Name: "__api_heap_sys", + Kind: MetricKindValue, + Description: "StatsHouse API bytes of heap memory obtained from the OS.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2317,10 +2743,14 @@ var BuiltinMetricMetaApiHeapSys = &MetricMetaValue{ const BuiltinMetricIDApiHeapIdle = -124 var BuiltinMetricMetaApiHeapIdle = &MetricMetaValue{ - Name: "__api_heap_idle", - Kind: MetricKindValue, - Description: "StatsHouse API bytes in idle (unused) spans.", - Resolution: 60, + Name: "__api_heap_idle", + Kind: MetricKindValue, + Description: "StatsHouse API bytes in idle (unused) spans.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2329,10 +2759,14 @@ var BuiltinMetricMetaApiHeapIdle = &MetricMetaValue{ const BuiltinMetricIDApiHeapInuse = -125 var BuiltinMetricMetaApiHeapInuse = &MetricMetaValue{ - Name: "__api_heap_inuse", - Kind: MetricKindValue, - Description: "StatsHouse API bytes in in-use spans.", - Resolution: 60, + Name: "__api_heap_inuse", + Kind: MetricKindValue, + Description: "StatsHouse API bytes in in-use spans.", + Resolution: 60, + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "host", }}, @@ -2341,10 +2775,14 @@ var BuiltinMetricMetaApiHeapInuse = &MetricMetaValue{ const BuiltinMetricIDClientWriteError = -126 var BuiltinMetricMetaClientWriteError = &MetricMetaValue{ - Name: "__src_client_write_err", - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "Bytes lost on StatsHouse clients.", + Name: "__src_client_write_err", + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "Bytes lost on StatsHouse clients.", + NoSampleAgent: false, + BuiltinAllowedToReceive: true, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "lang", ValueComments: convertToValueComments(map[int32]string{ @@ -2362,10 +2800,14 @@ var BuiltinMetricMetaClientWriteError = &MetricMetaValue{ const BuiltinMetricIDAgentTimings = -127 var BuiltinMetricMetaAgentTimings = &MetricMetaValue{ - Name: "__src_timings", - Kind: MetricKindValue, - Description: "Timings of agent operations", - MetricType: MetricNanosecond, + Name: "__src_timings", + Kind: MetricKindValue, + Description: "Timings of agent operations", + MetricType: MetricNanosecond, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: false, Tags: []MetricMetaTag{{ Description: "group", ValueComments: convertToValueComments(map[int32]string{ @@ -2397,9 +2839,13 @@ var BuiltinMetricMetaAgentTimings = &MetricMetaValue{ const BuiltinMetricIDAggBucketInfo = -128 var BuiltinMetricMetaAggBucketInfo = &MetricMetaValue{ - Name: "__agg_bucket_info", - Kind: MetricKindValue, - Description: `Statistics on received bucket`, + Name: "__agg_bucket_info", + Kind: MetricKindValue, + Description: `Statistics on received bucket`, + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: true, + WithAggregatorID: true, Tags: []MetricMetaTag{{ Description: "-", }, { @@ -2433,8 +2879,12 @@ var BuiltinMetricMetaAggBucketInfo = &MetricMetaValue{ const BuiltinMetricIDBudgetOwner = -129 //invisible, but owner mapping is flood-protected var BuiltinMetricMetaBudgetOwner = &MetricMetaValue{ - Name: "__budget_owner", - Kind: MetricKindCounter, - Description: "Invisible metric used only for accounting budget to create owner mappings", - Tags: []MetricMetaTag{}, + Name: "__budget_owner", + Kind: MetricKindCounter, + Description: "Invisible metric used only for accounting budget to create owner mappings", + NoSampleAgent: false, + BuiltinAllowedToReceive: false, + WithAgentEnvRouteArch: false, + WithAggregatorID: false, + Tags: []MetricMetaTag{}, } diff --git a/internal/format/builtin_tags.go b/internal/format/builtin_tags.go index c40f2c292..7aa595bba 100644 --- a/internal/format/builtin_tags.go +++ b/internal/format/builtin_tags.go @@ -131,6 +131,7 @@ const ( TagValueIDSrcIngestionStatusErrShardingFailed = 54 TagValueIDSrcIngestionStatusWarnTimestampClampedPast = 55 TagValueIDSrcIngestionStatusWarnTimestampClampedFuture = 56 + TagValueIDSrcIngestionStatusErrMetricBuiltin = 57 TagValueIDPacketFormatLegacy = 1 TagValueIDPacketFormatTL = 2 @@ -346,3 +347,35 @@ const ( TagValueIDAggBucketInfoUniqueBytes = 9 TagValueIDAggBucketInfoStringTops = 10 ) + +var ( + versionToValue = map[int32]string{ + 1: "v1", + 2: "v2", + } + + secondsToValue = map[int32]string{ + 1: "1s", + 5: "5s", + 15: "15s", + 60: "1m", + 300: "5m", + 900: "15m", + 3600: "1h", + 14400: "4h", + 86400: "24h", + 604800: "7d", + 2678400: "1M", + } + + apiCacheEvictionReason = map[int32]string{ + 1: "stale", // known to be stale + 2: "LRU", // evicted to free up memory + 3: "update", // evicted by more recent load + } + + apiBufferKind = map[int32]string{ + 1: "pool", // "sync.Pool", allocated buffer is subject for reuse (good) + 2: "heap", // large buffer won't be reused (bad, should not happen) + } +) diff --git a/internal/format/format.go b/internal/format/format.go index 8a4a7ae9f..42b3f327a 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -262,7 +262,6 @@ type MetricMetaValue struct { Sharding []MetricSharding `json:"sharding,omitempty"` PipelineVersion uint8 `json:"pipeline_version,omitempty"` - RawTagMask uint32 `json:"-"` // Should be restored from Tags after reading Name2Tag map[string]MetricMetaTag `json:"-"` // Should be restored from Tags after reading EffectiveResolution int `json:"-"` // Should be restored from Tags after reading PreKeyIndex int `json:"-"` // index of tag which goes to 'prekey' column, or <0 if no tag goes @@ -271,12 +270,16 @@ type MetricMetaValue struct { HasPercentiles bool `json:"-"` RoundSampleFactors bool `json:"-"` // Experimental, set if magic word in description is found ShardUniqueValues bool `json:"-"` // Experimental, set if magic word in description is found - NoSampleAgent bool `json:"-"` // Built-in metrics with fixed/limited # of rows on agent WhalesOff bool `json:"-"` // "whales" sampling algorithm disabled HistorgamBuckets []float32 `json:"-"` // Prometheus histogram buckets IsHardwareSlowMetric bool `json:"-"` GroupID int32 `json:"-"` + NoSampleAgent bool `json:"-"` // Built-in metrics with fixed/limited # of rows on agent. Set only in constant initialization of builtin metrics + BuiltinAllowedToReceive bool `json:"-"` // we allow only small subset of built-in metrics through agent receiver. + WithAgentEnvRouteArch bool `json:"-"` // set for some built-in metrics to add common set of tags + WithAggregatorID bool `json:"-"` // set for some built-in metrics to add common set of tags + Group *MetricsGroup `json:"-"` // don't use directly Namespace *NamespaceMeta `json:"-"` // don't use directly } @@ -383,7 +386,6 @@ func (m *MetricMetaValue) RestoreCachedInfo() error { err = multierr.Append(err, fmt.Errorf("invalid metric kind %q", m.Kind)) } - var mask uint32 m.Name2Tag = map[string]MetricMetaTag{} if m.StringTopName == StringTopTagID { // remove redundancy @@ -437,9 +439,6 @@ func (m *MetricMetaValue) RestoreCachedInfo() error { } for i := range tags { tag := &tags[i] - if tag.Raw { - mask |= 1 << i - } if len(tag.ID2Value) > 0 { // Legacy info, set for many metrics. Move to modern one. tag.ValueComments = convertToValueComments(tag.ID2Value) @@ -471,7 +470,6 @@ func (m *MetricMetaValue) RestoreCachedInfo() error { m.setName2Tag(TagID(i), MetricMetaTag{Index: i}, true, &err) } } - m.RawTagMask = mask m.EffectiveResolution = AllowedResolution(m.Resolution) if m.EffectiveResolution != m.Resolution { err = multierr.Append(err, fmt.Errorf("resolution %d must be factor of 60", m.Resolution)) @@ -511,7 +509,7 @@ func (m *MetricMetaValue) RestoreCachedInfo() error { } } } - m.NoSampleAgent = builtinMetricsNoSamplingAgent[m.MetricID] + // m.NoSampleAgent we never set it here, it is set in code for some built-in metrics if m.GroupID == 0 || m.GroupID == BuiltinGroupIDDefault { m.GroupID = BuiltinGroupIDDefault m.Group = BuiltInGroupDefault[BuiltinGroupIDDefault] diff --git a/internal/format/predefined_hardware.go b/internal/format/predefined_hardware.go index 06a8d9a05..b0d6d84bd 100644 --- a/internal/format/predefined_hardware.go +++ b/internal/format/predefined_hardware.go @@ -214,10 +214,11 @@ func HardwareMetric(metricID int32) bool { // add host tag later var hostMetrics = map[int32]*MetricMetaValue{ BuiltinMetricIDCPUUsage: { - Name: BuiltinMetricNameCpuUsage, - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "The number of seconds the CPU has spent performing different kinds of work", + Name: BuiltinMetricNameCpuUsage, + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "The number of seconds the CPU has spent performing different kinds of work", + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "state", Raw: true, @@ -241,10 +242,11 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDMemUsage: { - Name: BuiltinMetricNameMemUsage, - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "Amount of free and used memory in the system", + Name: BuiltinMetricNameMemUsage, + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "Amount of free and used memory in the system", + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "state", Raw: true, @@ -288,14 +290,16 @@ var hostMetrics = map[int32]*MetricMetaValue{ }}, }, BuiltinMetricIDProcessCreated: { - Name: BuiltinMetricNameProcessCreated, - Kind: MetricKindCounter, - Description: "Number of processes and threads created", + Name: BuiltinMetricNameProcessCreated, + Kind: MetricKindCounter, + Description: "Number of processes and threads created", + NoSampleAgent: true, }, BuiltinMetricIDProcessRunning: { - Name: BuiltinMetricNameProcessStatus, - Kind: MetricKindValue, - Description: "Number of processes currently blocked, waiting IO or running on CPUs", + Name: BuiltinMetricNameProcessStatus, + Kind: MetricKindValue, + Description: "Number of processes currently blocked, waiting IO or running on CPUs", + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "status", Raw: true, @@ -307,16 +311,18 @@ var hostMetrics = map[int32]*MetricMetaValue{ }}, }, BuiltinMetricIDSystemUptime: { - Name: BuiltinMetricNameSystemUptime, - Kind: MetricKindValue, - MetricType: MetricSecond, - Description: "The amount of time the system has been running", + Name: BuiltinMetricNameSystemUptime, + Kind: MetricKindValue, + MetricType: MetricSecond, + Description: "The amount of time the system has been running", + NoSampleAgent: true, }, BuiltinMetricIDPSICPU: { - Name: BuiltinMetricNamePSICPU, - Kind: MetricKindValue, - Description: "PSI of CPU", // todo fix + Name: BuiltinMetricNamePSICPU, + Kind: MetricKindValue, + Description: "PSI of CPU", // todo fix + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "type", Raw: true, @@ -327,9 +333,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }}, }, BuiltinMetricIDPSIMem: { - Name: BuiltinMetricNamePSIMem, - Kind: MetricKindValue, - Description: "PSI of memory", + Name: BuiltinMetricNamePSIMem, + Kind: MetricKindValue, + Description: "PSI of memory", + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "type", Raw: true, @@ -340,9 +347,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }}, }, BuiltinMetricIDPSIIO: { - Name: BuiltinMetricNamePSIIO, - Kind: MetricKindValue, - Description: "PSI of IO", + Name: BuiltinMetricNamePSIIO, + Kind: MetricKindValue, + Description: "PSI of IO", + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "type", Raw: true, @@ -354,10 +362,11 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, BuiltinMetricIDNetBandwidth: { - Name: BuiltinMetricNameNetBandwidth, - Kind: MetricKindMixed, - MetricType: MetricByte, - Description: "Total bandwidth of all physical network interfaces. Count - number of packets, Value - number of bytes", + Name: BuiltinMetricNameNetBandwidth, + Kind: MetricKindMixed, + MetricType: MetricByte, + Description: "Total bandwidth of all physical network interfaces. Count - number of packets, Value - number of bytes", + NoSampleAgent: true, Tags: []MetricMetaTag{{ Description: "type", Raw: true, @@ -429,9 +438,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDNetPacket: { - Name: BuiltinMetricNameNetPacket, - Kind: MetricKindCounter, - Description: "Number of transferred packets grouped by protocol", + Name: BuiltinMetricNameNetPacket, + Kind: MetricKindCounter, + Description: "Number of transferred packets grouped by protocol", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "type", @@ -454,9 +464,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDNetError: { - Name: BuiltinMetricNameNetError, - Kind: MetricKindCounter, - Description: "Number of network errors", + Name: BuiltinMetricNameNetError, + Kind: MetricKindCounter, + Description: "Number of network errors", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "type", @@ -491,10 +502,11 @@ var hostMetrics = map[int32]*MetricMetaValue{ }}, }, BuiltinMetricIDDiskUsage: { - Name: BuiltinMetricNameDiskUsage, - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "Disk space utilization", + Name: BuiltinMetricNameDiskUsage, + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "Disk space utilization", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "state", @@ -513,9 +525,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }}, }, BuiltinMetricIDINodeUsage: { - Name: BuiltinMetricNameINodeUsage, - Kind: MetricKindValue, - Description: "", + Name: BuiltinMetricNameINodeUsage, + Kind: MetricKindValue, + Description: "", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "state", @@ -535,9 +548,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, BuiltinMetricIDTCPSocketStatus: { - Name: BuiltinMetricNameTCPSocketStatus, - Kind: MetricKindValue, - Description: "The number of TCP socket grouped by state", + Name: BuiltinMetricNameTCPSocketStatus, + Kind: MetricKindValue, + Description: "The number of TCP socket grouped by state", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "state", @@ -552,17 +566,18 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDTCPSocketMemory: { - Name: BuiltinMetricNameTCPSocketMemory, - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "The amount of memory used by TCP sockets in all states", - Tags: []MetricMetaTag{}, + Name: BuiltinMetricNameTCPSocketMemory, + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "The amount of memory used by TCP sockets in all states", + NoSampleAgent: true, }, BuiltinMetricIDSocketMemory: { - Name: BuiltinMetricNameSocketMemory, - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "The amount of memory used by sockets", + Name: BuiltinMetricNameSocketMemory, + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "The amount of memory used by sockets", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "protocol", @@ -578,9 +593,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDSocketUsed: { - Name: BuiltinMetricNameSocketUsedv2, - Kind: MetricKindValue, - Description: "The number of socket in inuse state grouped by protocol", + Name: BuiltinMetricNameSocketUsedv2, + Kind: MetricKindValue, + Description: "The number of socket in inuse state grouped by protocol", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "protocol", @@ -596,9 +612,10 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDSoftIRQ: { - Name: BuiltinMetricNameSoftIRQ, // TODO add total time spend with eBPF - Kind: MetricKindValue, - Description: "Total number of software interrupts in the system", + Name: BuiltinMetricNameSoftIRQ, // TODO add total time spend with eBPF + Kind: MetricKindValue, + Description: "Total number of software interrupts in the system", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "type", @@ -619,22 +636,23 @@ var hostMetrics = map[int32]*MetricMetaValue{ }, }, BuiltinMetricIDIRQ: { - Name: BuiltinMetricNameIRQ, // TODO add total time spend with eBPF - Kind: MetricKindCounter, - Description: "Total number of interrupts in the system", - Tags: []MetricMetaTag{}, + Name: BuiltinMetricNameIRQ, // TODO add total time spend with eBPF + Kind: MetricKindCounter, + Description: "Total number of interrupts in the system", + NoSampleAgent: true, }, BuiltinMetricIDContextSwitch: { - Name: BuiltinMetricNameContextSwitch, - Kind: MetricKindCounter, - Description: "Total number of context switch in the system", - Tags: []MetricMetaTag{}, + Name: BuiltinMetricNameContextSwitch, + Kind: MetricKindCounter, + Description: "Total number of context switch in the system", + NoSampleAgent: true, }, BuiltinMetricIDWriteback: { - Name: BuiltinMetricNameWriteback, - Kind: MetricKindValue, - MetricType: MetricByte, - Description: "Writeback/Dirty memory", + Name: BuiltinMetricNameWriteback, + Kind: MetricKindValue, + MetricType: MetricByte, + Description: "Writeback/Dirty memory", + NoSampleAgent: true, Tags: []MetricMetaTag{ { Description: "type", diff --git a/internal/mapping/pipeline_v2.go b/internal/mapping/pipeline_v2.go index 250e321e4..2646b0aec 100644 --- a/internal/mapping/pipeline_v2.go +++ b/internal/mapping/pipeline_v2.go @@ -234,6 +234,8 @@ func MapErrorFromHeader(m tlstatshouse.MetricBytes, h data_model.MappedMetricHea return fmt.Errorf("not utf-8 metric name %q (hex) (envTag %d)", h.InvalidString, envTag) case format.TagValueIDSrcIngestionStatusErrMapTagNameEncoding: return fmt.Errorf("not utf-8 name %q (hex) for key of metric %q (envTag %d)", h.InvalidString, m.Name, envTag) + case format.TagValueIDSrcIngestionStatusErrMetricBuiltin: + return fmt.Errorf("metric %q is builtin (envTag %d)", m.Name, envTag) default: return fmt.Errorf("unexpected error status %d with invalid string value %q for key %q of metric %q (envTag %d)", h.IngestionStatus, h.InvalidString, ingestionTagName, m.Name, envTag) } diff --git a/internal/metajournal/journal.go b/internal/metajournal/journal.go index c08794e2c..9f8d64867 100644 --- a/internal/metajournal/journal.go +++ b/internal/metajournal/journal.go @@ -301,7 +301,9 @@ func (ms *Journal) updateJournal(aggLog AggLog) error { if len(newJournal) > 0 { lastEntry := newJournal[len(newJournal)-1] - log.Printf("Version updated from '%d' to '%d', last entity updated is %s, journal hash is '%s'", oldVersion, lastEntry.Version, lastEntry.Name, stateHash) + // TODO - remove this printf in tests + log.Printf("Version updated from '%d' to '%d', last entity updated is %s, journal hash is '%s'", + oldVersion, lastEntry.Version, lastEntry.Name, stateHash) if aggLog != nil { aggLog("journal_update", "", "ok", strconv.FormatInt(oldVersion, 10), diff --git a/internal/metajournal/meta_metrics_test.go b/internal/metajournal/meta_metrics_test.go index 93e4d8932..19e8b3e05 100644 --- a/internal/metajournal/meta_metrics_test.go +++ b/internal/metajournal/meta_metrics_test.go @@ -379,10 +379,11 @@ func TestMetricsStorage(t *testing.T) { // actually 1 test, but grouped by small test case (need to run together) t.Run("updateJournal test(each other depends on previous)", func(t *testing.T) { descrField := "__description" - metric := format.BuiltinMetrics[format.BuiltinMetricIDAPIBRS] + metric := *format.BuiltinMetrics[format.BuiltinMetricIDAPIBRS] metric.MetricID = metricID metric.Name = "test_metric" metric.GroupID = 0 + metric.BuiltinAllowedToReceive = false // this field is not restored by RestoreCachedInfo _ = metric.RestoreCachedInfo() metricBytes, err := metric.MarshalBinary() require.NoError(t, err) @@ -423,8 +424,8 @@ func TestMetricsStorage(t *testing.T) { require.Len(t, m.metricsByID, 1) require.Len(t, m.metricsByName, 1) - require.Equal(t, *metric, *m.metricsByID[metric.MetricID]) - require.Equal(t, *metric, *m.metricsByName[metric.Name]) + require.Equal(t, metric, *m.metricsByID[metric.MetricID]) + require.Equal(t, metric, *m.metricsByName[metric.Name]) }) t.Run("edit metric", func(t *testing.T) { @@ -452,8 +453,8 @@ func TestMetricsStorage(t *testing.T) { require.Len(t, m.metricsByID, 1) require.Len(t, m.metricsByName, 1) - require.Equal(t, *metric, *m.metricsByID[metric.MetricID]) - require.Equal(t, *metric, *m.metricsByName[metric.Name]) + require.Equal(t, metric, *m.metricsByID[metric.MetricID]) + require.Equal(t, metric, *m.metricsByName[metric.Name]) }) t.Run("rename metric", func(t *testing.T) { @@ -482,8 +483,8 @@ func TestMetricsStorage(t *testing.T) { require.Len(t, m.metricsByID, 1) require.Len(t, m.metricsByName, 1) - require.Equal(t, *metric, *m.metricsByID[metric.MetricID]) - require.Equal(t, *metric, *m.metricsByName[metric.Name]) + require.Equal(t, metric, *m.metricsByID[metric.MetricID]) + require.Equal(t, metric, *m.metricsByName[metric.Name]) }) t.Run("create dashboard", func(t *testing.T) { @@ -669,7 +670,7 @@ func TestMetricsStorage(t *testing.T) { }) t.Run("metric created (check new metric in group)", func(t *testing.T) { var id int32 = 65463 - metricCopy := *metric + metricCopy := metric metricCopy.Version = incVersion() metricCopy.MetricID = id metricCopy.Name = group1.Name + "_metric5"