From 41029729ac3b1de3a412137235ff7b729bd8a1ad Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 28 Nov 2024 10:43:40 +0100 Subject: [PATCH 1/6] Extract block-builder into its own module --- cmd/tempo/app/modules.go | 7 +-- .../ingest-storage/docker-compose.yaml | 60 ++++++++++++++----- .../docker-compose/ingest-storage/tempo.yaml | 8 ++- integration/e2e/ingest/config-kafka.yaml | 13 ++-- integration/e2e/ingest/kafka.go | 13 ---- modules/blockbuilder/blockbuilder.go | 12 ++-- modules/blockbuilder/blockbuilder_test.go | 6 +- modules/blockbuilder/config.go | 21 ++++++- modules/blockbuilder/util/id_test.go | 11 +++- modules/generator/config.go | 2 +- modules/storage/config.go | 3 +- tempodb/wal/wal.go | 5 ++ 12 files changed, 110 insertions(+), 51 deletions(-) diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go index 9acb9243ee8..45a7c9d25d4 100644 --- a/cmd/tempo/app/modules.go +++ b/cmd/tempo/app/modules.go @@ -81,8 +81,6 @@ const ( // composite targets SingleBinary string = "all" ScalableSingleBinary string = "scalable-single-binary" - // GeneratorBuilder is a composite target that includes the metrics-generator and the block-builder - GeneratorBuilder string = "generator-builder" // ring names ringIngester string = "ingester" @@ -673,15 +671,14 @@ func (t *App) setupModuleManager() error { QueryFrontend: {Common, Store, OverridesAPI}, Distributor: {Common, IngesterRing, MetricsGeneratorRing, PartitionRing}, Ingester: {Common, Store, MemberlistKV, PartitionRing}, - MetricsGenerator: {Common, OptionalStore, MemberlistKV, BlockBuilder, PartitionRing}, + MetricsGenerator: {Common, OptionalStore, MemberlistKV, PartitionRing}, Querier: {Common, Store, IngesterRing, MetricsGeneratorRing, SecondaryIngesterRing}, Compactor: {Common, Store, MemberlistKV}, BlockBuilder: {Common, Store, MemberlistKV, PartitionRing}, // composite targets - SingleBinary: {Compactor, QueryFrontend, Querier, Ingester, Distributor, MetricsGenerator}, + SingleBinary: {Compactor, QueryFrontend, Querier, Ingester, Distributor, MetricsGenerator, BlockBuilder}, ScalableSingleBinary: {SingleBinary}, - // GeneratorBuilder: {MetricsGenerator, BlockBuilder}, } for mod, targets := range deps { diff --git a/example/docker-compose/ingest-storage/docker-compose.yaml b/example/docker-compose/ingest-storage/docker-compose.yaml index cdf9c03bc1d..a8406be1f96 100644 --- a/example/docker-compose/ingest-storage/docker-compose.yaml +++ b/example/docker-compose/ingest-storage/docker-compose.yaml @@ -112,7 +112,8 @@ services: image: *tempoImage depends_on: - kafka - command: "-target=metrics-generator -config.file=/etc/tempo.yaml -block-builder.assigned-partitions=[0,1,2]" + command: "-target=metrics-generator -config.file=/etc/tempo.yaml" + hostname: metrics-generator-0 restart: always volumes: - ./tempo.yaml:/etc/tempo.yaml @@ -122,19 +123,50 @@ services: # environment: # - OTEL_EXPORTER_OTLP_ENDPOINT=http://alloy:4318 -# metrics-generator-1: -# image: *tempoImage -# depends_on: -# - kafka -# command: "-target=metrics-generator -config.file=/etc/tempo.yaml" -# restart: always -# volumes: -# - ./tempo.yaml:/etc/tempo.yaml -# ports: -# - "3200" # tempo -# # Uncomment the following lines to enable tracing -# # environment: -# # - OTEL_EXPORTER_OTLP_ENDPOINT=http://alloy:4318 + metrics-generator-1: + image: *tempoImage + depends_on: + - kafka + command: "-target=metrics-generator -config.file=/etc/tempo.yaml" + hostname: metrics-generator-1 + restart: always + volumes: + - ./tempo.yaml:/etc/tempo.yaml + ports: + - "3200" # tempo + # Uncomment the following lines to enable tracing + # environment: + # - OTEL_EXPORTER_OTLP_ENDPOINT=http://alloy:4318 + + block-builder-0: + image: *tempoImage + depends_on: + - kafka + command: "-target=block-builder -config.file=/etc/tempo.yaml" + hostname: block-builder-0 + restart: always + volumes: + - ./tempo.yaml:/etc/tempo.yaml + ports: + - "3200" # tempo + # Uncomment the following lines to enable tracing + # environment: + # - OTEL_EXPORTER_OTLP_ENDPOINT=http://alloy:4318 + + block-builder-1: + image: *tempoImage + depends_on: + - kafka + command: "-target=block-builder -config.file=/etc/tempo.yaml" + hostname: block-builder-1 + restart: always + volumes: + - ./tempo.yaml:/etc/tempo.yaml + ports: + - "3200" # tempo + # Uncomment the following lines to enable tracing + # environment: + # - OTEL_EXPORTER_OTLP_ENDPOINT=http://alloy:4318 minio: image: minio/minio:latest diff --git a/example/docker-compose/ingest-storage/tempo.yaml b/example/docker-compose/ingest-storage/tempo.yaml index b334a53d64f..e478ef4ec53 100644 --- a/example/docker-compose/ingest-storage/tempo.yaml +++ b/example/docker-compose/ingest-storage/tempo.yaml @@ -47,6 +47,9 @@ metrics_generator: send_exemplars: true traces_storage: path: /var/tempo/generator/traces + assigned_partitions: + metrics-generator-0: [0,2] + metrics-generator-1: [1] storage: trace: @@ -76,4 +79,7 @@ ingest: block_builder: lookback_on_no_commit: 2m - consume_cycle_duration: 30s \ No newline at end of file + consume_cycle_duration: 30s + assigned_partitions: + block-builder-0: [0,2] + block-builder-1: [1] \ No newline at end of file diff --git a/integration/e2e/ingest/config-kafka.yaml b/integration/e2e/ingest/config-kafka.yaml index 475a13de4ab..ec99ae09978 100644 --- a/integration/e2e/ingest/config-kafka.yaml +++ b/integration/e2e/ingest/config-kafka.yaml @@ -11,7 +11,6 @@ query_frontend: query_ingesters_until: 0 distributor: - kafka_write_path_enabled: true receivers: jaeger: protocols: @@ -34,10 +33,10 @@ ingester: replication_factor: 1 final_sleep: 0s trace_idle_period: 1s -# max_block_bytes: 1 -# max_block_duration: 2s -# complete_block_timeout: 20s -# flush_check_period: 1s + max_block_bytes: 1 + max_block_duration: 2s + complete_block_timeout: 20s + flush_check_period: 1s partition_ring: kvstore: store: inmemory @@ -64,6 +63,10 @@ overrides: local: path: /var/tempo/overrides +block_builder: + lookback_on_no_commit: 15m + consume_cycle_duration: 1m + ingest: enabled: true kafka: diff --git a/integration/e2e/ingest/kafka.go b/integration/e2e/ingest/kafka.go index be39258c384..0bd07c41bb6 100644 --- a/integration/e2e/ingest/kafka.go +++ b/integration/e2e/ingest/kafka.go @@ -4,19 +4,6 @@ import ( "github.com/grafana/e2e" ) -//- "CLUSTER_ID=zH1GDqcNTzGMDCXm5VZQdg" -//- "KAFKA_BROKER_ID=1" -//- "KAFKA_NUM_PARTITIONS=100" -//- "KAFKA_PROCESS_ROLES=broker,controller" -//- "KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,PLAINTEXT_HOST://:29092" -//- "KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092" -//- "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" -//- "KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT" -//- "KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER" -//- "KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka:9093" -//- "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1" -//- "KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=10000" - const kafkaImage = "confluentinc/cp-kafka:7.7.1" func NewKafka() *e2e.HTTPService { diff --git a/modules/blockbuilder/blockbuilder.go b/modules/blockbuilder/blockbuilder.go index cf7b13dce6a..2f7fbdc20ab 100644 --- a/modules/blockbuilder/blockbuilder.go +++ b/modules/blockbuilder/blockbuilder.go @@ -59,7 +59,6 @@ func New( decoder: ingest.NewDecoder(), overrides: overrides, writer: store, - wal: store.WAL(), } b.Service = services.NewBasicService(b.starting, b.running, b.stopping) @@ -70,13 +69,18 @@ func (b *BlockBuilder) starting(ctx context.Context) (err error) { level.Info(b.logger).Log("msg", "block builder starting") b.enc = encoding.DefaultEncoding() - if b.cfg.blockConfig.BlockCfg.Version != "" { - b.enc, err = encoding.FromVersion(b.cfg.blockConfig.BlockCfg.Version) + if version := b.cfg.BlockConfig.BlockCfg.Version; version != "" { + b.enc, err = encoding.FromVersion(version) if err != nil { return fmt.Errorf("failed to create encoding: %w", err) } } + b.wal, err = wal.New(&b.cfg.WAL) + if err != nil { + return fmt.Errorf("failed to create WAL: %w", err) + } + // Fallback offset is a millisecond timestamp used to look up a real offset if partition doesn't have a commit. b.fallbackOffsetMillis = time.Now().Add(-b.cfg.LookbackOnNoCommit).UnixMilli() @@ -228,7 +232,7 @@ func (b *BlockBuilder) consumePartitionSection(ctx context.Context, partition in ) // TODO - Review what endTimestamp is used here - writer := newPartitionSectionWriter(b.logger, int64(partition), sectionEndTime.UnixMilli(), b.cfg.blockConfig, b.overrides, b.wal, b.enc) + writer := newPartitionSectionWriter(b.logger, int64(partition), sectionEndTime.UnixMilli(), b.cfg.BlockConfig, b.overrides, b.wal, b.enc) // We always rewind the partition's offset to the commit offset by reassigning the partition to the client (this triggers partition assignment). // This is so the cycle started exactly at the commit offset, and not at what was (potentially over-) consumed previously. diff --git a/modules/blockbuilder/blockbuilder_test.go b/modules/blockbuilder/blockbuilder_test.go index d5a9d4131e9..bd65d8181cf 100644 --- a/modules/blockbuilder/blockbuilder_test.go +++ b/modules/blockbuilder/blockbuilder_test.go @@ -326,11 +326,11 @@ func TestCycleEndAtStartup(t *testing.T) { } } -func blockbuilderConfig(_ *testing.T, address string) Config { +func blockbuilderConfig(t *testing.T, address string) Config { cfg := Config{} flagext.DefaultValues(&cfg) - flagext.DefaultValues(&cfg.blockConfig) + flagext.DefaultValues(&cfg.BlockConfig) flagext.DefaultValues(&cfg.IngestStorageConfig.Kafka) cfg.IngestStorageConfig.Kafka.Address = address @@ -341,6 +341,8 @@ func blockbuilderConfig(_ *testing.T, address string) Config { cfg.LookbackOnNoCommit = 15 * time.Second cfg.ConsumeCycleDuration = 5 * time.Second + cfg.WAL.Filepath = t.TempDir() + return cfg } diff --git a/modules/blockbuilder/config.go b/modules/blockbuilder/config.go index 98456bacd32..8757779915a 100644 --- a/modules/blockbuilder/config.go +++ b/modules/blockbuilder/config.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/tempo/pkg/util/log" "github.com/grafana/tempo/tempodb/encoding" "github.com/grafana/tempo/tempodb/encoding/common" + "github.com/grafana/tempo/tempodb/wal" ) type BlockConfig struct { @@ -38,13 +39,26 @@ type Config struct { LookbackOnNoCommit time.Duration `yaml:"lookback_on_no_commit" category:"advanced"` - blockConfig BlockConfig `yaml:"block" doc:"Configuration for the block builder."` + BlockConfig BlockConfig `yaml:"block" doc:"Configuration for the block builder."` + WAL wal.Config `yaml:"wal" doc:"Configuration for the write ahead log."` // This config is dynamically injected because defined outside the ingester config. IngestStorageConfig ingest.Config `yaml:"-"` } func (c *Config) Validate() error { + if c.BlockConfig.BlockCfg.Version != c.WAL.Version { + return fmt.Errorf("block version %s does not match WAL version %s", c.BlockConfig.BlockCfg.Version, c.WAL.Version) + } + + if err := common.ValidateConfig(&c.BlockConfig.BlockCfg); err != nil { + return fmt.Errorf("block config validation failed: %w", err) + } + + if err := wal.ValidateConfig(&c.WAL); err != nil { + return fmt.Errorf("wal config validation failed: %w", err) + } + return nil } @@ -65,7 +79,10 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { // TODO - Review default f.DurationVar(&c.LookbackOnNoCommit, prefix+".lookback-on-no-commit", 12*time.Hour, "How much of the historical records to look back when there is no kafka commit for a partition.") - c.blockConfig.RegisterFlagsAndApplyDefaults(prefix+".block", f) + c.BlockConfig.RegisterFlagsAndApplyDefaults(prefix+".block", f) + c.WAL.RegisterFlags(f) + c.WAL.Version = c.BlockConfig.BlockCfg.Version + f.StringVar(&c.WAL.Filepath, prefix+".wal.path", "/var/tempo/block-builder/traces", "Path at which store WAL blocks.") } type partitionAssignmentVar struct { diff --git a/modules/blockbuilder/util/id_test.go b/modules/blockbuilder/util/id_test.go index e767b96c632..95635dd4c5d 100644 --- a/modules/blockbuilder/util/id_test.go +++ b/modules/blockbuilder/util/id_test.go @@ -6,12 +6,13 @@ import ( "github.com/google/uuid" "github.com/grafana/tempo/tempodb/backend" + "github.com/stretchr/testify/assert" ) func TestDeterministicIDGenerator(t *testing.T) { ts := time.Now().UnixMilli() - gen := NewDeterministicIDGenerator(ts) + gen := NewDeterministicIDGenerator(0, ts) firstPassIDs := make(map[backend.UUID]struct{}) for seq := int64(0); seq < 10; seq++ { @@ -19,7 +20,13 @@ func TestDeterministicIDGenerator(t *testing.T) { firstPassIDs[id] = struct{}{} } - gen = NewDeterministicIDGenerator(ts) + // Verify that that UUIDs are valid + for id := range firstPassIDs { + _, err := uuid.Parse(id.String()) + assert.NoError(t, err) + } + + gen = NewDeterministicIDGenerator(0, ts) for seq := int64(0); seq < 10; seq++ { id := gen.NewID() if _, ok := firstPassIDs[id]; !ok { diff --git a/modules/generator/config.go b/modules/generator/config.go index 5f55648f909..83fcfb5db8b 100644 --- a/modules/generator/config.go +++ b/modules/generator/config.go @@ -52,8 +52,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.Processor.RegisterFlagsAndApplyDefaults(prefix, f) cfg.Registry.RegisterFlagsAndApplyDefaults(prefix, f) cfg.Storage.RegisterFlagsAndApplyDefaults(prefix, f) + cfg.TracesWAL.RegisterFlags(f) cfg.TracesWAL.Version = encoding.DefaultEncoding().Version() - cfg.TracesWAL.IngestionSlack = 2 * time.Minute // setting default for max span age before discarding to 30s cfg.MetricsIngestionSlack = 30 * time.Second diff --git a/modules/storage/config.go b/modules/storage/config.go index 62766ec83a5..c7766a57c60 100644 --- a/modules/storage/config.go +++ b/modules/storage/config.go @@ -2,7 +2,6 @@ package storage import ( "flag" - "time" "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/util" @@ -35,10 +34,10 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) f.DurationVar(&cfg.Trace.BlocklistPoll, util.PrefixConfig(prefix, "trace.blocklist_poll"), tempodb.DefaultBlocklistPoll, "Period at which to run the maintenance cycle.") cfg.Trace.WAL = &wal.Config{} + cfg.Trace.WAL.RegisterFlags(f) f.StringVar(&cfg.Trace.WAL.Filepath, util.PrefixConfig(prefix, "trace.wal.path"), "/var/tempo/wal", "Path at which store WAL blocks.") cfg.Trace.WAL.Encoding = backend.EncSnappy cfg.Trace.WAL.SearchEncoding = backend.EncNone - cfg.Trace.WAL.IngestionSlack = 2 * time.Minute cfg.Trace.Search = &tempodb.SearchConfig{} cfg.Trace.Search.RegisterFlagsAndApplyDefaults(prefix, f) diff --git a/tempodb/wal/wal.go b/tempodb/wal/wal.go index 055bbb61f5d..6b7c1595fc1 100644 --- a/tempodb/wal/wal.go +++ b/tempodb/wal/wal.go @@ -1,6 +1,7 @@ package wal import ( + "flag" "fmt" "os" "path/filepath" @@ -32,6 +33,10 @@ type Config struct { Version string `yaml:"version,omitempty"` } +func (c *Config) RegisterFlags(*flag.FlagSet) { + c.IngestionSlack = 2 * time.Minute +} + func ValidateConfig(c *Config) error { if _, err := encoding.FromVersion(c.Version); err != nil { return fmt.Errorf("failed to validate block version %s: %w", c.Version, err) From 5949bd3f7ea62ce278c30d35b3b81a1f1405490c Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 2 Dec 2024 12:37:32 +0100 Subject: [PATCH 2/6] Update /operations and examples --- example/tk/jsonnetfile.lock.json | 18 +- example/tk/lib/k.libsonnet | 2 +- .../load-generator.json | 630 ------------------ .../synthetic-load-generator/main.libsonnet | 22 +- example/tk/tempo-microservices/main.jsonnet | 14 +- .../grafana-builder/grafana.libsonnet | 187 +++++- .../microservices/block-builder.libsonnet | 69 ++ .../jsonnet/microservices/config.libsonnet | 13 + .../jsonnet/microservices/configmap.libsonnet | 7 + .../jsonnet/microservices/tempo.libsonnet | 1 + 10 files changed, 284 insertions(+), 679 deletions(-) delete mode 100644 example/tk/lib/synthetic-load-generator/load-generator.json create mode 100644 operations/jsonnet/microservices/block-builder.libsonnet diff --git a/example/tk/jsonnetfile.lock.json b/example/tk/jsonnetfile.lock.json index ef53d72468e..0a67309ded3 100644 --- a/example/tk/jsonnetfile.lock.json +++ b/example/tk/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "grafana" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "Y5nheroSOIwmE+djEVPq4OvvTxKenzdHhpEwaR3Ebjs=" }, { @@ -18,8 +18,8 @@ "subdir": "grafana-builder" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", - "sum": "B49EzIY2WZsFxNMJcgRxE/gcZ9ltnS8pkOOV6Q5qioc=" + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", + "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo=" }, { "source": { @@ -28,7 +28,7 @@ "subdir": "ksonnet-util" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "0y3AFX9LQSpfWTxWKSwoLgbt0Wc9nnCwhMH2szKzHv0=" }, { @@ -38,7 +38,7 @@ "subdir": "kube-state-metrics/" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "q1YzD+I4InDdfQP6k93W9Lw5U1jpll3gVaS9rUzLR7U=" }, { @@ -48,7 +48,7 @@ "subdir": "memcached" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "Cc715Y3rgTuimgDFIw+FaKzXSJGRYwt1pFTMbdrNBD8=" }, { @@ -58,7 +58,7 @@ "subdir": "prometheus" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "b4scQI+UtFxkVgzzD3dfbPMwI7EVtyjv1uETEmizM8M=" }, { @@ -68,7 +68,7 @@ "subdir": "tanka-util" } }, - "version": "84e49c8549fa472c963862f233422c8b368afabe", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "ShSIissXdvCy1izTCDZX6tY7qxCoepE5L+WJ52Hw7ZQ=" }, { @@ -88,7 +88,7 @@ "subdir": "1.29" } }, - "version": "bf9a62cfd32a58c071b8410bfcdec058475dd25e", + "version": "6ecbb7709baf27f44b2e48f3529741ae6754ae6a", "sum": "i2w3hGbgQmaB73t5LJHSioPOVdYv8ZBvivHiDwZJVyI=" } ], diff --git a/example/tk/lib/k.libsonnet b/example/tk/lib/k.libsonnet index 8f0a057f9e6..a5f12df6c95 100644 --- a/example/tk/lib/k.libsonnet +++ b/example/tk/lib/k.libsonnet @@ -1 +1 @@ -import '1.21/main.libsonnet' +import '1.29/main.libsonnet' diff --git a/example/tk/lib/synthetic-load-generator/load-generator.json b/example/tk/lib/synthetic-load-generator/load-generator.json deleted file mode 100644 index 58a2dac3e08..00000000000 --- a/example/tk/lib/synthetic-load-generator/load-generator.json +++ /dev/null @@ -1,630 +0,0 @@ -{ - "topology": { - "services": [ - { - "serviceName": "frontend", - "tagSets": [ - { - "weight": 1, - "tags": { - "version": "v127", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - }, - { - "weight": 1, - "tags": { - "version": "v125", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - }, - { - "weight": 2, - "tags": { - "version": "v125", - "region": "us-west-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ], - "routes": [ - { - "route": "/product", - "downstreamCalls": { - "productcatalogservice": "/GetProducts", - "recommendationservice": "/GetRecommendations", - "adservice": "/AdRequest" - }, - "tagSets": [ - { - "weight": 1, - "tags": { - "starter": "charmander" - }, - "tagGenerators": [ - { - "rand": { - "seed": 179867746078676, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - }, - "tagGen": {}, - "valLength": 16, - "numTags": 50, - "numVals": 3000 - } - ], - "inherit": [] - }, - { - "weight": 1, - "tags": { - "starter": "squirtle" - }, - "tagGenerators": [], - "inherit": [] - }, - { - "weight": 1, - "tags": { - "starter": "bulbasaur" - }, - "tagGenerators": [], - "inherit": [] - } - ] - }, - { - "route": "/cart", - "downstreamCalls": { - "cartservice": "/GetCart", - "recommendationservice": "/GetRecommendations" - }, - "tagSets": [] - }, - { - "route": "/checkout", - "downstreamCalls": { - "checkoutservice": "/PlaceOrder" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 800 - } - ] - }, - { - "route": "/shipping", - "downstreamCalls": { - "shippingservice": "/GetQuote" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 50 - } - ] - }, - { - "route": "/currency", - "downstreamCalls": { - "currencyservice": "/GetConversion" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 50 - } - ] - } - ], - "instances": [ - "frontend-6b654dbf57-zq8dt", - "frontend-d847fdcf5-j6s2f", - "frontend-79d8c8d6c8-9sbff" - ], - "mergedTagSets": {}, - "random": { - "seed": 187004238864083, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "productcatalogservice", - "tagSets": [ - { - "tags": { - "version": "v52" - }, - "tagGenerators": [], - "inherit": [ - "region" - ] - } - ], - "routes": [ - { - "route": "/GetProducts", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [ - "starter" - ], - "maxLatency": 100 - } - ] - }, - { - "route": "/SearchProducts", - "downstreamCalls": {}, - "tagSets": [ - { - "weight": 15, - "tags": { - "error": true, - "http.status_code": 503 - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 400 - }, - { - "weight": 85, - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 400 - } - ] - } - ], - "instances": [ - "productcatalogservice-6b654dbf57-zq8dt", - "productcatalogservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 238238032670139, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "recommendationservice", - "tagSets": [ - { - "tags": { - "version": "v234", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetRecommendations", - "downstreamCalls": { - "productcatalogservice": "/GetProducts" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 200 - } - ] - } - ], - "instances": [ - "recommendationservice-6b654dbf57-zq8dt", - "recommendationservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 66295214032801, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "cartservice", - "tagSets": [ - { - "tags": { - "version": "v5", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetCart", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 200 - } - ] - } - ], - "instances": [ - "cartservice-6b654dbf57-zq8dt", - "cartservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 234194353561392, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "checkoutservice", - "tagSets": [ - { - "tags": { - "version": "v37", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ], - "routes": [ - { - "route": "/PlaceOrder", - "downstreamCalls": { - "paymentservice": "/CreditCardInfo", - "shippingservice": "/Address", - "currencyservice": "/GetConversion", - "cartservice": "/GetCart", - "emailservice": "/SendOrderConfirmation" - }, - "tagSets": [ - { - "weight": 25, - "tags": { - "error": true, - "http.status_code": 503 - }, - "tagGenerators": [], - "inherit": [] - }, - { - "weight": 85, - "tags": {}, - "tagGenerators": [], - "inherit": [] - } - ] - } - ], - "instances": [ - "checkoutservice-6b654dbf57-zq8dt", - "checkoutservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 60782549660568, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "paymentservice", - "tagSets": [ - { - "tags": { - "version": "v177", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/ChargeRequest", - "downstreamCalls": { - "paymentservice": "/CreditCardInfo" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 700 - } - ] - }, - { - "route": "/CreditCardInfo", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 50 - } - ] - } - ], - "instances": [ - "paymentservice-6b654dbf57-zq8dt", - "paymentservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 174850031049111, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "shippingservice", - "tagSets": [ - { - "tags": { - "version": "v127", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetQuote", - "downstreamCalls": { - "shippingservice": "/Address" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 250 - } - ] - }, - { - "route": "/ShipOrder", - "downstreamCalls": { - "shippingservice": "/Address" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ] - }, - { - "route": "/Address", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - } - ], - "instances": [ - "shippingservice-6b654dbf57-zq8dt", - "shippingservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 107892261530518, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "emailservice", - "tagSets": [ - { - "tags": { - "version": "v27", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ], - "routes": [ - { - "route": "/SendOrderConfirmation", - "downstreamCalls": { - "emailservice": "/OrderResult" - }, - "tagSets": [ - { - "weight": 15, - "tags": { - "error": true, - "http.status_code": 503 - }, - "tagGenerators": [], - "inherit": [] - }, - { - "weight": 85, - "tags": {}, - "tagGenerators": [], - "inherit": [] - } - ] - }, - { - "route": "/OrderResult", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - } - ], - "instances": [ - "emailservice-6b654dbf57-zq8dt", - "emailservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 61175057559946, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "currencyservice", - "tagSets": [ - { - "tags": { - "version": "v27", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetConversion", - "downstreamCalls": { - "currencyservice": "/Money" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - }, - { - "route": "/Money", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - } - ], - "instances": [ - "currencyservice-6b654dbf57-zq8dt", - "currencyservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 66219471499700, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "adservice", - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ], - "routes": [ - { - "route": "/AdRequest", - "downstreamCalls": {}, - "tagSets": [] - }, - { - "route": "/Ad", - "downstreamCalls": {}, - "tagSets": [] - } - ], - "instances": [ - "adservice-6b654dbf57-zq8dt", - "adservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 22694143111805, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - } - ] - }, - "rootRoutes": [ - { - "service": "frontend", - "route": "/product", - "tracesPerHour": 2880 - }, - { - "service": "frontend", - "route": "/cart", - "tracesPerHour": 14400 - }, - { - "service": "frontend", - "route": "/shipping", - "tracesPerHour": 480 - }, - { - "service": "frontend", - "route": "/currency", - "tracesPerHour": 200 - }, - { - "service": "frontend", - "route": "/checkout", - "tracesPerHour": 480 - } - ] -} \ No newline at end of file diff --git a/example/tk/lib/synthetic-load-generator/main.libsonnet b/example/tk/lib/synthetic-load-generator/main.libsonnet index cb902fa1be2..63294a42048 100644 --- a/example/tk/lib/synthetic-load-generator/main.libsonnet +++ b/example/tk/lib/synthetic-load-generator/main.libsonnet @@ -1,33 +1,17 @@ { local k = import 'ksonnet-util/kausal.libsonnet', - local configMap = k.core.v1.configMap, local container = k.core.v1.container, - local volumeMount = k.core.v1.volumeMount, local deployment = k.apps.v1.deployment, - local volume = k.core.v1.volume, - - synthetic_load_generator_configmap: - configMap.new('synthetic-load-generator') + - configMap.withData({ - 'load-generator.json': importstr './load-generator.json', - }), synthetic_load_generator_container:: - container.new('synthetic-load-gen', 'omnition/synthetic-load-generator:1.0.25') + - container.withVolumeMounts([ - volumeMount.new('conf', '/conf'), - ]) + + container.new('synthetic-load-gen', 'ghcr.io/grafana/xk6-client-tracing:v0.0.5') + container.withEnvMap({ - TOPOLOGY_FILE: '/conf/load-generator.json', - JAEGER_COLLECTOR_URL: 'http://tempo:14268', + ENDPOINT: 'http://tempo:4317', }), synthetic_load_generator_deployment: deployment.new('synthetic-load-generator', 1, [ $.synthetic_load_generator_container ], - { app: 'synthetic_load_generator' }) + - deployment.mixin.spec.template.spec.withVolumes([ - volume.fromConfigMap('conf', $.synthetic_load_generator_configmap.metadata.name), - ]), + { app: 'synthetic_load_generator' }), } diff --git a/example/tk/tempo-microservices/main.jsonnet b/example/tk/tempo-microservices/main.jsonnet index 5cc02f9cc95..5f2f6820145 100644 --- a/example/tk/tempo-microservices/main.jsonnet +++ b/example/tk/tempo-microservices/main.jsonnet @@ -26,17 +26,22 @@ minio + metrics + load + tempo { }, distributor+: { receivers: { - opencensus: null, - jaeger: { + otlp: { protocols: { - thrift_http: null, + grpc: null, }, }, }, }, metrics_generator+: { + replicas: 1, ephemeral_storage_limit_size: '2Gi', ephemeral_storage_request_size: '1Gi', + pvc_size: '1Gi', + pvc_storage_class: 'local-path', + }, + block_builder+:{ + replicas: 1, }, memcached+: { replicas: 1, @@ -79,8 +84,7 @@ minio + metrics + load + tempo { tempo_distributor_container+:: k.util.resourcesRequests('500m', '500Mi') + container.withPortsMixin([ - containerPort.new('opencensus', 55678), - containerPort.new('jaeger-http', 14268), + containerPort.new('otlp-grpc', 4317), ]), tempo_ingester_container+:: diff --git a/example/tk/vendor/github.com/grafana/jsonnet-libs/grafana-builder/grafana.libsonnet b/example/tk/vendor/github.com/grafana/jsonnet-libs/grafana-builder/grafana.libsonnet index 0bd0b339493..be43616c8d3 100644 --- a/example/tk/vendor/github.com/grafana/jsonnet-libs/grafana-builder/grafana.libsonnet +++ b/example/tk/vendor/github.com/grafana/jsonnet-libs/grafana-builder/grafana.libsonnet @@ -1,3 +1,5 @@ +local utils = import 'mixin-utils/utils.libsonnet'; + { dashboard(title, uid='', datasource='default', datasource_regex=''):: { // Stuff that isn't materialised. @@ -41,7 +43,7 @@ }, }, - addMultiTemplate(name, metric_name, label_name, hide=0, allValue='.+', sort=2):: self { + addMultiTemplate(name, metric_name, label_name, hide=0, allValue='.+', sort=2, includeAll=true):: self { templating+: { list+: [{ allValue: allValue, @@ -52,7 +54,7 @@ }, datasource: '$datasource', hide: hide, - includeAll: true, + includeAll: includeAll, label: name, multi: true, name: name, @@ -70,6 +72,40 @@ }, }, + addShowNativeLatencyVariable():: self { + templating+: { + list+: [{ + current: { + selected: true, + text: 'classic', + value: '1', + }, + description: 'Choose between showing latencies based on low precision classic or high precision native histogram metrics.', + hide: 0, + includeAll: false, + label: 'Latency metrics', + multi: false, + name: 'latency_metrics', + query: 'native : -1,classic : 1', + options: [ + { + selected: false, + text: 'native', + value: '-1', + }, + { + selected: true, + text: 'classic', + value: '1', + }, + ], + skipUrlSync: false, + type: 'custom', + useTags: false, + }], + }, + }, + dashboardLinkUrl(title, url):: self { links+: [ { @@ -296,17 +332,25 @@ }, statPanel(query, format='percentunit'):: { + local isNativeClassic = utils.isNativeClassicQuery(query), type: 'singlestat', thresholds: '70,80', format: format, targets: [ { - expr: query, + expr: if isNativeClassic then utils.showClassicHistogramQuery(query) else query, + format: 'time_series', + instant: true, + refId: if isNativeClassic then 'A_classic' else 'A', + }, + ] + if isNativeClassic then [ + { + expr: utils.showNativeHistogramQuery(query), format: 'time_series', instant: true, refId: 'A', }, - ], + ] else [], }, tablePanel(queries, labelStyles):: { @@ -420,18 +464,20 @@ }, ], + httpStatusColors:: { + '1xx': '#EAB839', + '2xx': '#7EB26D', + '3xx': '#6ED0E0', + '4xx': '#EF843C', + '5xx': '#E24D42', + OK: '#7EB26D', + success: '#7EB26D', + 'error': '#E24D42', + cancel: '#A9A9A9', + }, + qpsPanel(selector, statusLabelName='status_code'):: { - aliasColors: { - '1xx': '#EAB839', - '2xx': '#7EB26D', - '3xx': '#6ED0E0', - '4xx': '#EF843C', - '5xx': '#E24D42', - OK: '#7EB26D', - success: '#7EB26D', - 'error': '#E24D42', - cancel: '#A9A9A9', - }, + aliasColors: $.httpStatusColors, targets: [ { expr: @@ -448,6 +494,65 @@ ], } + $.stack, + // Assumes that the metricName is for a histogram (as opposed to qpsPanel above) + // Assumes that there is a dashboard variable named latency_metrics, values are -1 (native) or 1 (classic) + qpsPanelNativeHistogram(metricName, selector, statusLabelName='status_code'):: { + local sumByStatus(nativeClassicQuery) = { + local template = + ||| + sum by (status) ( + label_replace(label_replace(%(metricQuery)s, + "status", "${1}xx", "%(label)s", "([0-9]).."), + "status", "${1}", "%(label)s", "([a-zA-Z]+)")) + |||, + native: template % { metricQuery: nativeClassicQuery.native, label: statusLabelName }, + classic: template % { metricQuery: nativeClassicQuery.classic, label: statusLabelName }, + }, + fieldConfig+: { + defaults+: { + custom+: { + lineWidth: 0, + fillOpacity: 100, // Get solid fill. + stacking: { + mode: 'normal', + group: 'A', + }, + }, + unit: 'reqps', + min: 0, + }, + overrides+: [{ + matcher: { + id: 'byName', + options: status, + }, + properties: [ + { + id: 'color', + value: { + mode: 'fixed', + fixedColor: $.httpStatusColors[status], + }, + }, + ], + } for status in std.objectFieldsAll($.httpStatusColors)], + }, + targets: [ + { + expr: utils.showClassicHistogramQuery(sumByStatus(utils.ncHistogramCountRate(metricName, selector))), + format: 'time_series', + legendFormat: '{{status}}', + refId: 'A_classic', + }, + { + expr: utils.showNativeHistogramQuery(sumByStatus(utils.ncHistogramCountRate(metricName, selector))), + format: 'time_series', + legendFormat: '{{status}}', + refId: 'A', + }, + ], + } + $.stack, + latencyPanel(metricName, selector, multiplier='1e3'):: { nullPointMode: 'null as zero', targets: [ @@ -473,6 +578,58 @@ yaxes: $.yaxes('ms'), }, + // Assumes that there is a dashboard variable named latency_metrics, values are -1 (native) or 1 (classic) + latencyPanelNativeHistogram(metricName, selector, multiplier='1e3'):: { + nullPointMode: 'null as zero', + fieldConfig+: { + defaults+: { + custom+: { + fillOpacity: 10, + }, + unit: 'ms', + }, + }, + targets: [ + { + expr: utils.showNativeHistogramQuery(utils.ncHistogramQuantile('0.99', metricName, selector, multiplier=multiplier)), + format: 'time_series', + legendFormat: '99th percentile', + refId: 'A', + }, + { + expr: utils.showClassicHistogramQuery(utils.ncHistogramQuantile('0.99', metricName, selector, multiplier=multiplier)), + format: 'time_series', + legendFormat: '99th percentile', + refId: 'A_classic', + }, + { + expr: utils.showNativeHistogramQuery(utils.ncHistogramQuantile('0.50', metricName, selector, multiplier=multiplier)), + format: 'time_series', + legendFormat: '50th percentile', + refId: 'B', + }, + { + expr: utils.showClassicHistogramQuery(utils.ncHistogramQuantile('0.50', metricName, selector, multiplier=multiplier)), + format: 'time_series', + legendFormat: '50th percentile', + refId: 'B_classic', + }, + { + expr: utils.showNativeHistogramQuery(utils.ncHistogramAverageRate(metricName, selector, multiplier=multiplier)), + format: 'time_series', + legendFormat: 'Average', + refId: 'C', + }, + { + expr: utils.showClassicHistogramQuery(utils.ncHistogramAverageRate(metricName, selector, multiplier=multiplier)), + format: 'time_series', + legendFormat: 'Average', + refId: 'C_classic', + }, + ], + yaxes: $.yaxes('ms'), + }, + selector:: { eq(label, value):: { label: label, op: '=', value: value }, neq(label, value):: { label: label, op: '!=', value: value }, diff --git a/operations/jsonnet/microservices/block-builder.libsonnet b/operations/jsonnet/microservices/block-builder.libsonnet new file mode 100644 index 00000000000..ecd395c1683 --- /dev/null +++ b/operations/jsonnet/microservices/block-builder.libsonnet @@ -0,0 +1,69 @@ +{ + local k = import 'ksonnet-util/kausal.libsonnet', + + local container = k.core.v1.container, + local containerPort = k.core.v1.containerPort, + local volumeMount = k.core.v1.volumeMount, + local statefulset = k.apps.v1.statefulSet, + local volume = k.core.v1.volume, + local envVar = k.core.v1.envVar, + local configMap = k.core.v1.configMap, + + local target_name = 'block-builder', + local tempo_config_volume = 'tempo-conf', + local tempo_overrides_config_volume = 'overrides', + + // Statefulset + + tempo_block_builder_ports:: [containerPort.new('prom-metrics', $._config.port)], + + tempo_block_builder_args:: { + target: target_name, + 'config.file': '/conf/tempo.yaml', + 'mem-ballast-size-mbs': $._config.ballast_size_mbs, + }, + + tempo_block_builder_container:: + container.new(target_name, $._images.tempo) + + container.withPorts($.tempo_block_builder_ports) + + container.withArgs($.util.mapToFlags($.tempo_block_builder_args)) + + container.withVolumeMounts([ + volumeMount.new(tempo_config_volume, '/conf'), + volumeMount.new(tempo_overrides_config_volume, '/overrides'), + ]) + + $.util.withResources($._config.block_builder.resources) + + (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) + + container.mixin.resources.withRequestsMixin({ 'ephemeral-storage': $._config.block_builder.ephemeral_storage_request_size }) + + container.mixin.resources.withLimitsMixin({ 'ephemeral-storage': $._config.block_builder.ephemeral_storage_limit_size }) + + $.util.readinessProbe + + (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}), + + tempo_block_builder_statefulset: + statefulset.new(target_name, $._config.block_builder.replicas, $.tempo_block_builder_container, [], { app: target_name }) + + statefulset.mixin.spec.withServiceName(target_name) + + statefulset.spec.template.spec.securityContext.withFsGroup(10001) + // 10001 is the UID of the tempo user + // statefulset.mixin.spec.strategy.rollingUpdate.withMaxSurge(3) + + // statefulset.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + + statefulset.mixin.spec.template.metadata.withAnnotations({ + config_hash: std.md5(std.toString($.tempo_block_builder_configmap.data['tempo.yaml'])), + }) + + statefulset.mixin.spec.template.spec.withVolumes([ + volume.fromConfigMap(tempo_config_volume, $.tempo_block_builder_configmap.metadata.name), + volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + ]) + + statefulset.mixin.spec.withPodManagementPolicy('Parallel'), + + // Configmap + + tempo_block_builder_configmap: + configMap.new('tempo-block-builder') + + configMap.withData({ + 'tempo.yaml': k.util.manifestYaml($.tempo_block_builder_config), + }), + + // Service + + tempo_block_builder_service: + k.util.serviceFor($.tempo_block_builder_statefulset) + +} diff --git a/operations/jsonnet/microservices/config.libsonnet b/operations/jsonnet/microservices/config.libsonnet index f54214555ed..7d0a018ce73 100644 --- a/operations/jsonnet/microservices/config.libsonnet +++ b/operations/jsonnet/microservices/config.libsonnet @@ -107,6 +107,19 @@ }, }, }, + block_builder: { + replicas: 0, + resources: { + requests: { + cpu: '500m', + memory: '1Gi', + }, + limits: { + cpu: '1', + memory: '2Gi', + }, + }, + }, memcached: { replicas: 3, connection_limit: 4096, diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index c2f33c0ccba..8379a851e4a 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -111,6 +111,7 @@ }, tempo_query_frontend_config:: $.tempo_config {}, + tempo_block_builder_config:: $.tempo_config {}, // This will be the single configmap that stores `overrides.yaml`. overrides_config: @@ -151,6 +152,12 @@ 'tempo.yaml': $.util.manifestYaml($.tempo_querier_config), }), + tempo_block_builder_configmap: + configMap.new('tempo-block-builder') + + configMap.withData({ + 'tempo.yaml': $.util.manifestYaml($.tempo_block_builder_config), + }), + tempo_query_frontend_configmap: configMap.new('tempo-query-frontend') + configMap.withData({ diff --git a/operations/jsonnet/microservices/tempo.libsonnet b/operations/jsonnet/microservices/tempo.libsonnet index af421491493..aa7c0782a61 100644 --- a/operations/jsonnet/microservices/tempo.libsonnet +++ b/operations/jsonnet/microservices/tempo.libsonnet @@ -7,6 +7,7 @@ (import 'generator.libsonnet') + (import 'frontend.libsonnet') + (import 'querier.libsonnet') + +(import 'block-builder.libsonnet') + (import 'vulture.libsonnet') + (import 'memcached.libsonnet') + (import 'multi-zone.libsonnet') + From 4a2baa7d4948d937b436264800387df66de0a984 Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 2 Dec 2024 12:42:01 +0100 Subject: [PATCH 3/6] No ephemeral storage --- operations/jsonnet/microservices/block-builder.libsonnet | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/operations/jsonnet/microservices/block-builder.libsonnet b/operations/jsonnet/microservices/block-builder.libsonnet index ecd395c1683..a7369f5e228 100644 --- a/operations/jsonnet/microservices/block-builder.libsonnet +++ b/operations/jsonnet/microservices/block-builder.libsonnet @@ -33,8 +33,6 @@ ]) + $.util.withResources($._config.block_builder.resources) + (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) + - container.mixin.resources.withRequestsMixin({ 'ephemeral-storage': $._config.block_builder.ephemeral_storage_request_size }) + - container.mixin.resources.withLimitsMixin({ 'ephemeral-storage': $._config.block_builder.ephemeral_storage_limit_size }) + $.util.readinessProbe + (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}), @@ -42,8 +40,8 @@ statefulset.new(target_name, $._config.block_builder.replicas, $.tempo_block_builder_container, [], { app: target_name }) + statefulset.mixin.spec.withServiceName(target_name) + statefulset.spec.template.spec.securityContext.withFsGroup(10001) + // 10001 is the UID of the tempo user - // statefulset.mixin.spec.strategy.rollingUpdate.withMaxSurge(3) + - // statefulset.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + + statefulset.mixin.spec.strategy.rollingUpdate.withMaxSurge(3) + + statefulset.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + statefulset.mixin.spec.template.metadata.withAnnotations({ config_hash: std.md5(std.toString($.tempo_block_builder_configmap.data['tempo.yaml'])), }) + From 241711929cde14e556a3f0369edb9033bf2bc30c Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 2 Dec 2024 12:43:09 +0100 Subject: [PATCH 4/6] No rolling strategy either --- operations/jsonnet/microservices/block-builder.libsonnet | 2 -- 1 file changed, 2 deletions(-) diff --git a/operations/jsonnet/microservices/block-builder.libsonnet b/operations/jsonnet/microservices/block-builder.libsonnet index a7369f5e228..005ad089fc3 100644 --- a/operations/jsonnet/microservices/block-builder.libsonnet +++ b/operations/jsonnet/microservices/block-builder.libsonnet @@ -40,8 +40,6 @@ statefulset.new(target_name, $._config.block_builder.replicas, $.tempo_block_builder_container, [], { app: target_name }) + statefulset.mixin.spec.withServiceName(target_name) + statefulset.spec.template.spec.securityContext.withFsGroup(10001) + // 10001 is the UID of the tempo user - statefulset.mixin.spec.strategy.rollingUpdate.withMaxSurge(3) + - statefulset.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + statefulset.mixin.spec.template.metadata.withAnnotations({ config_hash: std.md5(std.toString($.tempo_block_builder_configmap.data['tempo.yaml'])), }) + From d9afb091df73ff9c0f6a3d46c6a3b53085e0f2d6 Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 2 Dec 2024 13:06:06 +0100 Subject: [PATCH 5/6] fmt and compile --- Makefile | 2 +- .../ConfigMap-tempo-block-builder.yaml | 44 +++++++++++++ .../Service-block-builder.yaml | 15 +++++ .../StatefulSet-block-builder.yaml | 61 +++++++++++++++++++ .../util/jsonnetfile.lock.json | 4 +- .../microservices/block-builder.libsonnet | 2 +- 6 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 operations/jsonnet-compiled/ConfigMap-tempo-block-builder.yaml create mode 100644 operations/jsonnet-compiled/Service-block-builder.yaml create mode 100644 operations/jsonnet-compiled/StatefulSet-block-builder.yaml diff --git a/Makefile b/Makefile index 97de1af6eea..df7726eec09 100644 --- a/Makefile +++ b/Makefile @@ -340,7 +340,7 @@ jsonnet-test: tools-image ## Test jsonnet .PHONY: docker-serverless test-serverless docker-serverless: ## Build docker Tempo serverless $(MAKE) -C cmd/tempo-serverless build-docker - + test-serverless: ## Run Tempo serverless tests $(MAKE) -C cmd/tempo-serverless test diff --git a/operations/jsonnet-compiled/ConfigMap-tempo-block-builder.yaml b/operations/jsonnet-compiled/ConfigMap-tempo-block-builder.yaml new file mode 100644 index 00000000000..1230d1780be --- /dev/null +++ b/operations/jsonnet-compiled/ConfigMap-tempo-block-builder.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +data: + tempo.yaml: | + compactor: {} + distributor: {} + http_api_prefix: "" + ingester: + lifecycler: + ring: + replication_factor: 3 + memberlist: + abort_if_cluster_join_fails: false + bind_port: 7946 + join_members: + - dns+gossip-ring.tracing.svc.cluster.local.:7946 + overrides: + per_tenant_override_config: /overrides/overrides.yaml + server: + http_listen_port: 3200 + storage: + trace: + azure: + container_name: tempo + backend: gcs + blocklist_poll: "0" + cache: memcached + gcs: + bucket_name: tempo + chunk_buffer_size: 1.048576e+07 + memcached: + consistent_hash: true + host: memcached + service: memcached-client + timeout: 200ms + pool: + queue_depth: 2000 + s3: + bucket: tempo + wal: + path: /var/tempo/wal +kind: ConfigMap +metadata: + name: tempo-block-builder + namespace: tracing diff --git a/operations/jsonnet-compiled/Service-block-builder.yaml b/operations/jsonnet-compiled/Service-block-builder.yaml new file mode 100644 index 00000000000..607c0e60725 --- /dev/null +++ b/operations/jsonnet-compiled/Service-block-builder.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: block-builder + name: block-builder + namespace: tracing +spec: + ports: + - name: block-builder-prom-metrics + port: 3200 + targetPort: 3200 + selector: + app: block-builder + name: block-builder diff --git a/operations/jsonnet-compiled/StatefulSet-block-builder.yaml b/operations/jsonnet-compiled/StatefulSet-block-builder.yaml new file mode 100644 index 00000000000..a718577689f --- /dev/null +++ b/operations/jsonnet-compiled/StatefulSet-block-builder.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: block-builder + namespace: tracing +spec: + podManagementPolicy: Parallel + replicas: 0 + selector: + matchLabels: + app: block-builder + name: block-builder + serviceName: block-builder + template: + metadata: + annotations: + config_hash: 46188d18f0d8adfa8586e9dbeb744db2 + labels: + app: block-builder + name: block-builder + spec: + containers: + - args: + - -config.file=/conf/tempo.yaml + - -mem-ballast-size-mbs=1024 + - -target=block-builder + image: grafana/tempo:latest + imagePullPolicy: IfNotPresent + name: block-builder + ports: + - containerPort: 3200 + name: prom-metrics + readinessProbe: + httpGet: + path: /ready + port: 3200 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - mountPath: /conf + name: tempo-conf + - mountPath: /overrides + name: overrides + securityContext: + fsGroup: 10001 + volumes: + - configMap: + name: tempo-block-builder + name: tempo-conf + - configMap: + name: tempo-overrides + name: overrides + updateStrategy: + type: RollingUpdate diff --git a/operations/jsonnet-compiled/util/jsonnetfile.lock.json b/operations/jsonnet-compiled/util/jsonnetfile.lock.json index b0837149e4e..a38267a88cc 100644 --- a/operations/jsonnet-compiled/util/jsonnetfile.lock.json +++ b/operations/jsonnet-compiled/util/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "ksonnet-util" } }, - "version": "da81b340bbb149e56bef63103d4b59de76590ce3", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "0y3AFX9LQSpfWTxWKSwoLgbt0Wc9nnCwhMH2szKzHv0=" }, { @@ -18,7 +18,7 @@ "subdir": "memcached" } }, - "version": "da81b340bbb149e56bef63103d4b59de76590ce3", + "version": "57b0b85dc1d7ed5e30c2e41f3bd26744b8aef519", "sum": "Cc715Y3rgTuimgDFIw+FaKzXSJGRYwt1pFTMbdrNBD8=" }, { diff --git a/operations/jsonnet/microservices/block-builder.libsonnet b/operations/jsonnet/microservices/block-builder.libsonnet index 005ad089fc3..926180fd2ab 100644 --- a/operations/jsonnet/microservices/block-builder.libsonnet +++ b/operations/jsonnet/microservices/block-builder.libsonnet @@ -60,6 +60,6 @@ // Service tempo_block_builder_service: - k.util.serviceFor($.tempo_block_builder_statefulset) + k.util.serviceFor($.tempo_block_builder_statefulset), } From b1cfc2afc5ab7e9f9c8c6d3a2064ca517ad432fd Mon Sep 17 00:00:00 2001 From: Mario Date: Tue, 3 Dec 2024 14:13:10 +0100 Subject: [PATCH 6/6] Address review comment --- cmd/tempo/app/config.go | 9 +++++++++ modules/blockbuilder/config.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/tempo/app/config.go b/cmd/tempo/app/config.go index 757f0d6d7f0..efb21c051a2 100644 --- a/cmd/tempo/app/config.go +++ b/cmd/tempo/app/config.go @@ -229,6 +229,10 @@ func (c *Config) CheckConfig() []ConfigWarning { warnings = append(warnings, warnTraceByIDConcurrentShards) } + if c.BlockBuilder.BlockConfig.BlockCfg.Version != c.BlockBuilder.WAL.Version { + warnings = append(warnings, warnBlockAndWALVersionMismatch) + } + return warnings } @@ -308,6 +312,11 @@ var ( Message: "c.Frontend.TraceByID.ConcurrentShards greater than query_shards is invalid. concurrent_shards will be set to query_shards", Explain: "Please remove ConcurrentShards or set it to a value less than or equal to QueryShards", } + + warnBlockAndWALVersionMismatch = ConfigWarning{ + Message: "c.BlockConfig.BlockCfg.Version != c.WAL.Version", + Explain: "Block version and WAL version must match. WAL version will be set to block version", + } ) func newV2Warning(setting string) ConfigWarning { diff --git a/modules/blockbuilder/config.go b/modules/blockbuilder/config.go index 8757779915a..f20ce9a2439 100644 --- a/modules/blockbuilder/config.go +++ b/modules/blockbuilder/config.go @@ -48,7 +48,7 @@ type Config struct { func (c *Config) Validate() error { if c.BlockConfig.BlockCfg.Version != c.WAL.Version { - return fmt.Errorf("block version %s does not match WAL version %s", c.BlockConfig.BlockCfg.Version, c.WAL.Version) + c.WAL.Version = c.BlockConfig.BlockCfg.Version } if err := common.ValidateConfig(&c.BlockConfig.BlockCfg); err != nil {