diff --git a/CHANGELOG.md b/CHANGELOG.md index bbf9175ee48..ccbcc962114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ storage: ``` * [CHANGE] Ignore context canceled errors in the queriers [#2440](https://github.com/grafana/tempo/pull/2440) (@joe-elliott) +* [CHANGE] Start flush queue worker after wal replay and block rediscovery [#2456](https://github.com/grafana/tempo/pull/2456) (@ie-pham) ## v2.1.1 / 2023-04-28 * [BUGFIX] Fix issue where Tempo sometimes flips booleans from false->true at storage time. [#2400](https://github.com/grafana/tempo/issues/2400) (@joe-elliott) diff --git a/modules/ingester/ingester.go b/modules/ingester/ingester.go index 9af716bfd78..50cd2445299 100644 --- a/modules/ingester/ingester.go +++ b/modules/ingester/ingester.go @@ -81,11 +81,6 @@ func New(cfg Config, store storage.Store, limits *overrides.Overrides, reg prome i.local = store.WAL().LocalBackend() - i.flushQueuesDone.Add(cfg.ConcurrentFlushes) - for j := 0; j < cfg.ConcurrentFlushes; j++ { - go i.flushLoop(j) - } - lc, err := ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", cfg.OverrideRingKey, true, log.Logger, prometheus.WrapRegistererWithPrefix("tempo_", reg)) if err != nil { return nil, fmt.Errorf("NewLifecycler failed: %w", err) @@ -114,6 +109,11 @@ func (i *Ingester) starting(ctx context.Context) error { return fmt.Errorf("failed to rediscover local blocks: %w", err) } + i.flushQueuesDone.Add(i.cfg.ConcurrentFlushes) + for j := 0; j < i.cfg.ConcurrentFlushes; j++ { + go i.flushLoop(j) + } + // Now that user states have been created, we can start the lifecycler. // Important: we want to keep lifecycler running until we ask it to stop, so we need to give it independent context if err := i.lifecycler.StartAsync(context.Background()); err != nil {