Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gbuteyko/statshouse work #1652

Merged
merged 4 commits into from
Jan 12, 2025
Merged

Gbuteyko/statshouse work #1652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d54223c
refactor builtin metrics
hrissan Jan 11, 2025
93e26cd
simplified built in metrics
hrissan Jan 11, 2025
0cb0d9c
tiny fix
hrissan Jan 11, 2025
5422594
fix to tests
hrissan Jan 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions cmd/statshouse/worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,6 @@ func (w *worker) HandleMetrics(args data_model.HandlerArgs) (h data_model.Mapped
w.fillTime(args, &h)
metaOk := w.fillMetricMeta(args, &h)
if metaOk {
h.Key.Metric = h.MetricMeta.MetricID
if !h.MetricMeta.Visible {
h.IngestionStatus = format.TagValueIDSrcIngestionStatusErrMetricInvisible
}
done = w.mapper.Map(args, h.MetricMeta, &h)
} else {
w.mapper.MapEnvironment(args.MetricBytes, &h)
Expand Down Expand Up @@ -95,11 +91,21 @@ func (w *worker) fillMetricMeta(args data_model.HandlerArgs, h *data_model.Mappe
metricMeta := w.metricStorage.GetMetaMetricByNameBytes(metric.Name)
if metricMeta != nil {
h.MetricMeta = metricMeta
h.Key.Metric = metricMeta.MetricID
if !metricMeta.Visible {
h.IngestionStatus = format.TagValueIDSrcIngestionStatusErrMetricInvisible
return false
}
return true
}
metricMeta = format.BuiltinMetricAllowedToReceive[string(metric.Name)]
metricMeta = format.BuiltinMetricByName[string(metric.Name)]
if metricMeta != nil {
h.MetricMeta = metricMeta
h.Key.Metric = metricMeta.MetricID
if !metricMeta.BuiltinAllowedToReceive {
h.IngestionStatus = format.TagValueIDSrcIngestionStatusErrMetricBuiltin
return false
}
return true
}

Expand Down
229 changes: 7 additions & 222 deletions internal/format/builtin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,6 @@ const (
var (
BuiltinMetricByName map[string]*MetricMetaValue

// list of built-in metrics which can be sent as normal metric. Used by API and prometheus exporter
// Description: "-" marks tags used in incompatible way in the past. We should not reuse such tags, because there would be garbage in historic data.
BuiltinMetricAllowedToReceive map[string]*MetricMetaValue

BuiltinMetrics = map[int32]*MetricMetaValue{
BuiltinMetricIDAgentSamplingFactor: BuiltinMetricMetaAgentSamplingFactor,
BuiltinMetricIDAggBucketReceiveDelaySec: BuiltinMetricMetaAggBucketReceiveDelaySec,
Expand Down Expand Up @@ -160,193 +156,14 @@ var (
BuiltinMetricIDBudgetOwner: BuiltinMetricMetaBudgetOwner,
}

// this set is very small, and we do not want to set Visible property for hunderds of metrics
builtinMetricsInvisible = map[int32]bool{
BuiltinMetricIDBudgetHost: true,
BuiltinMetricIDBudgetAggregatorHost: true,
BuiltinMetricIDBudgetUnknownMetric: true,
BuiltinMetricIDBudgetOwner: true,
}

// API and metadata sends this metrics via local statshouse instance
builtinMetricsAllowedToReceive = map[int32]bool{
BuiltinMetricIDTimingErrors: true,
BuiltinMetricIDPromScrapeTime: true,
BuiltinMetricIDAPIBRS: true,
BuiltinMetricIDAPIServiceTime: true,
BuiltinMetricIDAPIResponseTime: true,
BuiltinMetricIDUsageMemory: true,
BuiltinMetricIDUsageCPU: true,
BuiltinMetricIDAPIActiveQueries: true,
BuiltinMetricIDAPISelectRows: true,
BuiltinMetricIDAPISelectBytes: true,
BuiltinMetricIDAPISelectDuration: true,
BuiltinMetricIDSystemMetricScrapeDuration: true,
BuiltinMetricIDMetaServiceTime: true,
BuiltinMetricIDMetaClientWaits: true,
BuiltinMetricIDAPIMetricUsage: true,
BuiltinMetricIDHeartbeatVersion: true,
BuiltinMetricIDUIErrors: true,
BuiltinMetricIDStatsHouseErrors: true,
BuiltinMetricIDPromQLEngineTime: true,
BuiltinMetricIDAPICacheHit: true,
BuiltinMetricIDAPICacheBytesAlloc: true,
BuiltinMetricIDAPICacheBytesFree: true,
BuiltinMetricIDAPICacheBytesTotal: true,
BuiltinMetricIDAPICacheAgeEvict: true,
BuiltinMetricIDAPICacheAgeTotal: true,
BuiltinMetricIDAPIBufferBytesAlloc: true,
BuiltinMetricIDAPIBufferBytesFree: true,
BuiltinMetricIDAPIBufferBytesTotal: true,
BuiltinMetricIDAutoCreateMetric: true,
BuiltinMetricIDGCDuration: true,
BuiltinMetricIDProxyAcceptHandshakeError: true,
BuiltinMetricIDProxyVmSize: true,
BuiltinMetricIDProxyVmRSS: true,
BuiltinMetricIDProxyHeapAlloc: true,
BuiltinMetricIDProxyHeapSys: true,
BuiltinMetricIDProxyHeapIdle: true,
BuiltinMetricIDProxyHeapInuse: true,
BuiltinMetricIDApiVmSize: true,
BuiltinMetricIDApiVmRSS: true,
BuiltinMetricIDApiHeapAlloc: true,
BuiltinMetricIDApiHeapSys: true,
BuiltinMetricIDApiHeapIdle: true,
BuiltinMetricIDApiHeapInuse: true,
BuiltinMetricIDClientWriteError: true,
}

builtinMetricsNoSamplingAgent = map[int32]bool{
BuiltinMetricIDAgentMapping: true,
BuiltinMetricIDJournalVersions: true,
BuiltinMetricIDAgentReceivedPacketSize: true,
BuiltinMetricIDAgentReceivedBatchSize: true,
BuiltinMetricIDHeartbeatVersion: true,
BuiltinMetricIDHeartbeatArgs: true,
BuiltinMetricIDAgentDiskCacheErrors: true,
BuiltinMetricIDTimingErrors: true,
BuiltinMetricIDUsageMemory: true,
BuiltinMetricIDUsageCPU: true,

BuiltinMetricIDCPUUsage: true,
BuiltinMetricIDMemUsage: true,
BuiltinMetricIDProcessCreated: true,
BuiltinMetricIDProcessRunning: true,
BuiltinMetricIDSystemUptime: true,
BuiltinMetricIDPSICPU: true,
BuiltinMetricIDPSIMem: true,
BuiltinMetricIDPSIIO: true,
BuiltinMetricIDNetBandwidth: true,
BuiltinMetricIDNetPacket: true,
BuiltinMetricIDNetError: true,
BuiltinMetricIDDiskUsage: true,
BuiltinMetricIDINodeUsage: true,
BuiltinMetricIDTCPSocketStatus: true,
BuiltinMetricIDTCPSocketMemory: true,
BuiltinMetricIDSocketMemory: true,
BuiltinMetricIDSocketUsed: true,
BuiltinMetricIDSoftIRQ: true,
BuiltinMetricIDIRQ: true,
BuiltinMetricIDContextSwitch: true,
BuiltinMetricIDWriteback: true,
}

MetricsWithAgentEnvRouteArch = map[int32]bool{
BuiltinMetricIDAgentDiskCacheErrors: true,
BuiltinMetricIDTimingErrors: true,
BuiltinMetricIDAgentMapping: true,
BuiltinMetricIDAutoConfig: true, // also passed through ingress proxies
BuiltinMetricIDJournalVersions: true,
BuiltinMetricIDTLByteSizePerInflightType: true,
BuiltinMetricIDIngestionStatus: true,
BuiltinMetricIDAgentReceivedBatchSize: true,
BuiltinMetricIDAgentReceivedPacketSize: true,
BuiltinMetricIDAggSizeCompressed: true,
BuiltinMetricIDAggSizeUncompressed: true,
BuiltinMetricIDAggBucketReceiveDelaySec: true,
BuiltinMetricIDAggBucketAggregateTimeSec: true,
BuiltinMetricIDAggAdditionsToEstimator: true,
BuiltinMetricIDAgentHistoricQueueSize: true,
BuiltinMetricIDVersions: true,
BuiltinMetricIDAggKeepAlive: true,
BuiltinMetricIDAggMappingCreated: true,
BuiltinMetricIDUsageMemory: true,
BuiltinMetricIDUsageCPU: true,
BuiltinMetricIDHeartbeatVersion: true,
BuiltinMetricIDHeartbeatArgs: true,
BuiltinMetricIDAgentUDPReceiveBufferSize: true,
BuiltinMetricIDAgentDiskCacheSize: true,
BuiltinMetricIDSrcTestConnection: true,
BuiltinMetricIDAgentAggregatorTimeDiff: true,
BuiltinMetricIDSrcSamplingMetricCount: true,
BuiltinMetricIDSrcSamplingSizeBytes: true,
BuiltinMetricIDStatsHouseErrors: true,
BuiltinMetricIDSrcSamplingBudget: true,
BuiltinMetricIDSrcSamplingGroupBudget: true,
BuiltinMetricIDRestartTimings: true,
BuiltinMetricIDGCDuration: true,
BuiltinMetricIDAgentTimings: true,
BuiltinMetricIDAggBucketInfo: true,
}

metricsWithoutAggregatorID = map[int32]bool{
BuiltinMetricIDTLByteSizePerInflightType: true,
BuiltinMetricIDIngestionStatus: true,
BuiltinMetricIDAgentDiskCacheErrors: true,
BuiltinMetricIDAgentReceivedBatchSize: true,
BuiltinMetricIDAgentMapping: true,
BuiltinMetricIDAgentReceivedPacketSize: true,
BuiltinMetricIDBadges: true,
BuiltinMetricIDPromScrapeTime: true,
BuiltinMetricIDGeneratorConstCounter: true,
BuiltinMetricIDGeneratorSinCounter: true,
BuiltinMetricIDAPIBRS: true,
BuiltinMetricIDAPISelectRows: true,
BuiltinMetricIDAPISelectBytes: true,
BuiltinMetricIDAPISelectDuration: true,
BuiltinMetricIDAPIServiceTime: true,
BuiltinMetricIDAPIResponseTime: true,
BuiltinMetricIDAPIActiveQueries: true,
BuiltinMetricIDBudgetHost: true,
BuiltinMetricIDBudgetAggregatorHost: true,
BuiltinMetricIDSystemMetricScrapeDuration: true,
BuiltinMetricIDAgentUDPReceiveBufferSize: true,
BuiltinMetricIDAgentDiskCacheSize: true,
BuiltinMetricIDAPIMetricUsage: true,
BuiltinMetricIDSrcSamplingMetricCount: true,
BuiltinMetricIDSrcSamplingSizeBytes: true,
BuiltinMetricIDSrcSamplingBudget: true,
BuiltinMetricIDSrcSamplingGroupBudget: true,
BuiltinMetricIDUIErrors: true,
BuiltinMetricIDStatsHouseErrors: true,
BuiltinMetricIDPromQLEngineTime: true,
BuiltinMetricIDAPICacheBytesAlloc: true,
BuiltinMetricIDAPICacheBytesFree: true,
BuiltinMetricIDAPICacheBytesTotal: true,
BuiltinMetricIDAPICacheAgeEvict: true,
BuiltinMetricIDAPICacheAgeTotal: true,
BuiltinMetricIDAPIBufferBytesAlloc: true,
BuiltinMetricIDAPIBufferBytesFree: true,
BuiltinMetricIDAPIBufferBytesTotal: true,
BuiltinMetricIDRestartTimings: true,
BuiltinMetricIDGCDuration: true,
BuiltinMetricIDProxyAcceptHandshakeError: true,
BuiltinMetricIDProxyVmSize: true,
BuiltinMetricIDProxyVmRSS: true,
BuiltinMetricIDProxyHeapAlloc: true,
BuiltinMetricIDProxyHeapSys: true,
BuiltinMetricIDProxyHeapIdle: true,
BuiltinMetricIDProxyHeapInuse: true,
BuiltinMetricIDApiVmSize: true,
BuiltinMetricIDApiVmRSS: true,
BuiltinMetricIDApiHeapAlloc: true,
BuiltinMetricIDApiHeapSys: true,
BuiltinMetricIDApiHeapIdle: true,
BuiltinMetricIDApiHeapInuse: true,
BuiltinMetricIDClientWriteError: true,
BuiltinMetricIDAgentTimings: true,
BuiltinMetricIDBudgetOwner: true,
}

// BuiltInGroupDefault can be overridden by journal, don't use directly
BuiltInGroupDefault = map[int32]*MetricsGroup{
BuiltinGroupIDDefault: {
Expand All @@ -373,36 +190,6 @@ var (
Weight: 1,
},
}

versionToValue = map[int32]string{
1: "v1",
2: "v2",
}

secondsToValue = map[int32]string{
1: "1s",
5: "5s",
15: "15s",
60: "1m",
300: "5m",
900: "15m",
3600: "1h",
14400: "4h",
86400: "24h",
604800: "7d",
2678400: "1M",
}

apiCacheEvictionReason = map[int32]string{
1: "stale", // known to be stale
2: "LRU", // evicted to free up memory
3: "update", // evicted by more recent load
}

apiBufferKind = map[int32]string{
1: "pool", // "sync.Pool", allocated buffer is subject for reuse (good)
2: "heap", // large buffer won't be reused (bad, should not happen)
}
)

func TagIDTagToTagID(tagIDTag int32) string {
Expand Down Expand Up @@ -433,8 +220,10 @@ func init() {
v.Group = BuiltInGroupDefault[BuiltinGroupIDHost]
v.Sharding = []MetricSharding{{Strategy: ShardByMetric}}
BuiltinMetrics[k] = v
builtinMetricsAllowedToReceive[k] = true
metricsWithoutAggregatorID[k] = true
// v.NoSampleAgent = false
v.BuiltinAllowedToReceive = true
// v.WithAgentEnvRouteArch = false
// v.WithAggregatorID = false
}
for i := 0; i < NewMaxTags; i++ {
name := strconv.Itoa(i)
Expand All @@ -454,7 +243,6 @@ func init() {
tagIDTag2TagID[TagIDShift-2] = tagStringForUI + " " + HostTagID // for UI only

BuiltinMetricByName = make(map[string]*MetricMetaValue, len(BuiltinMetrics))
BuiltinMetricAllowedToReceive = make(map[string]*MetricMetaValue, len(BuiltinMetrics))
for id, m := range BuiltinMetrics {
m.MetricID = id
if m.GroupID == 0 {
Expand All @@ -467,9 +255,6 @@ func init() {

BuiltinMetricByName[m.Name] = m

if builtinMetricsAllowedToReceive[m.MetricID] {
BuiltinMetricAllowedToReceive[m.Name] = m
}
if id == BuiltinMetricIDIngestionStatus || id == BuiltinMetricIDAggMappingCreated {
m.Tags = append([]MetricMetaTag{{Description: "environment"}}, m.Tags...)
} else {
Expand All @@ -478,7 +263,7 @@ func init() {
for len(m.Tags) < MaxTags {
m.Tags = append(m.Tags, MetricMetaTag{Description: "-"})
}
if !metricsWithoutAggregatorID[id] {
if m.WithAggregatorID {
m.Tags[AggHostTag] = MetricMetaTag{Description: "aggregator_host"}
m.Tags[AggShardTag] = MetricMetaTag{Description: "aggregator_shard", Raw: true}
m.Tags[AggReplicaTag] = MetricMetaTag{Description: "aggregator_replica", Raw: true}
Expand All @@ -491,7 +276,7 @@ func init() {
m.Tags[HostOwnerTag] = MetricMetaTag{Description: "owner"}

}
if MetricsWithAgentEnvRouteArch[id] {
if m.WithAgentEnvRouteArch {
m.Tags[RouteTag] = MetricMetaTag{Description: "route", ValueComments: convertToValueComments(routeToValue)}
m.Tags[AgentEnvTag] = MetricMetaTag{
Description: "statshouse_env",
Expand Down
Loading
Loading