From 295ad96cbbd6116846b95a541cc2af5f1e4af103 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 20 May 2024 10:02:43 +0200 Subject: [PATCH 1/5] feature: Boilerplate for new bloom build planner component --- pkg/bloombuild/bloomplanner/config.go | 21 +++++++++++ pkg/bloombuild/bloomplanner/metrics.go | 26 ++++++++++++++ pkg/bloombuild/bloomplanner/planner.go | 48 ++++++++++++++++++++++++++ pkg/bloombuild/config.go | 32 +++++++++++++++++ pkg/loki/loki.go | 5 +++ pkg/loki/modules.go | 18 +++++++++- 6 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 pkg/bloombuild/bloomplanner/config.go create mode 100644 pkg/bloombuild/bloomplanner/metrics.go create mode 100644 pkg/bloombuild/bloomplanner/planner.go create mode 100644 pkg/bloombuild/config.go diff --git a/pkg/bloombuild/bloomplanner/config.go b/pkg/bloombuild/bloomplanner/config.go new file mode 100644 index 0000000000000..b6246461502d9 --- /dev/null +++ b/pkg/bloombuild/bloomplanner/config.go @@ -0,0 +1,21 @@ +package bloomplanner + +import "flag" + +// Config configures the bloom-planner component. +type Config struct { + // TODO: Add config +} + +// RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. +func (cfg *Config) RegisterFlagsWithPrefix(flagsPrefix string, f *flag.FlagSet) { + // TODO: Register flags with flagsPrefix +} + +func (cfg *Config) Validate() error { + return nil +} + +type Limits interface { + // TODO: Add limits +} diff --git a/pkg/bloombuild/bloomplanner/metrics.go b/pkg/bloombuild/bloomplanner/metrics.go new file mode 100644 index 0000000000000..45f99b5b3eef6 --- /dev/null +++ b/pkg/bloombuild/bloomplanner/metrics.go @@ -0,0 +1,26 @@ +package bloomplanner + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricsNamespace = "loki" + metricsSubsystem = "bloomplanner" +) + +type Metrics struct { + running prometheus.Gauge +} + +func NewMetrics(r prometheus.Registerer) *Metrics { + return &Metrics{ + running: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "running", + Help: "Value will be 1 if planner is currently running on this instance", + }), + } +} diff --git a/pkg/bloombuild/bloomplanner/planner.go b/pkg/bloombuild/bloomplanner/planner.go new file mode 100644 index 0000000000000..c1e38092c6774 --- /dev/null +++ b/pkg/bloombuild/bloomplanner/planner.go @@ -0,0 +1,48 @@ +package bloomplanner + +import ( + "context" + "github.com/go-kit/log" + "github.com/grafana/dskit/services" + utillog "github.com/grafana/loki/v3/pkg/util/log" + "github.com/prometheus/client_golang/prometheus" +) + +type Planner struct { + services.Service + + cfg Config + metrics *Metrics + logger log.Logger +} + +func New( + cfg Config, + logger log.Logger, + r prometheus.Registerer, +) (*Planner, error) { + utillog.WarnExperimentalUse("Bloom Build Planner", logger) + + p := &Planner{ + cfg: cfg, + metrics: NewMetrics(r), + logger: logger, + } + + p.Service = services.NewBasicService(p.starting, p.running, p.stopping) + return p, nil +} + +func (p *Planner) starting(_ context.Context) (err error) { + p.metrics.running.Set(1) + return err +} + +func (p *Planner) stopping(_ error) error { + p.metrics.running.Set(0) + return nil +} + +func (p *Planner) running(_ context.Context) error { + return nil +} diff --git a/pkg/bloombuild/config.go b/pkg/bloombuild/config.go new file mode 100644 index 0000000000000..3e1a1babecd0f --- /dev/null +++ b/pkg/bloombuild/config.go @@ -0,0 +1,32 @@ +package bloombuild + +import ( + "flag" + "fmt" + bloomplanner "github.com/grafana/loki/v3/pkg/bloombuild/planner" +) + +// Config configures the bloom-planner component. +type Config struct { + Enabled bool `yaml:"enabled"` + + Planner bloomplanner.Config `yaml:"planner"` +} + +// RegisterFlags registers flags for the bloom building configuration. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.BoolVar(&cfg.Enabled, "bloom-build.enabled", false, "Flag to enable or disable the usage of the bloom-build-planner and bloom-builder components.") + cfg.Planner.RegisterFlagsWithPrefix("bloom-build.enabled.planner", f) +} + +func (cfg *Config) Validate() error { + if !cfg.Enabled { + return nil + } + + if err := cfg.Planner.Validate(); err != nil { + return fmt.Errorf("invalid bloom planner configuration: %w", err) + } + + return nil +} diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index b682c4bfaa65c..fcbc6d81ca24d 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" "github.com/grafana/loki/v3/pkg/analytics" + "github.com/grafana/loki/v3/pkg/bloombuild" "github.com/grafana/loki/v3/pkg/bloomcompactor" "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" @@ -90,6 +91,7 @@ type Config struct { Pattern pattern.Config `yaml:"pattern_ingester,omitempty"` IndexGateway indexgateway.Config `yaml:"index_gateway"` BloomCompactor bloomcompactor.Config `yaml:"bloom_compactor,omitempty" category:"experimental"` + BloomBuild bloombuild.Config `yaml:"bloom_build,omitempty" category:"experimental"` BloomGateway bloomgateway.Config `yaml:"bloom_gateway,omitempty" category:"experimental"` StorageConfig storage.Config `yaml:"storage_config,omitempty"` ChunkStoreConfig config.ChunkStoreConfig `yaml:"chunk_store_config,omitempty"` @@ -173,6 +175,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Tracing.RegisterFlags(f) c.CompactorConfig.RegisterFlags(f) c.BloomCompactor.RegisterFlags(f) + c.BloomBuild.RegisterFlags(f) c.QueryScheduler.RegisterFlags(f) c.Analytics.RegisterFlags(f) c.OperationalConfig.RegisterFlags(f) @@ -649,6 +652,7 @@ func (t *Loki) setupModuleManager() error { mm.RegisterModule(BloomStore, t.initBloomStore) mm.RegisterModule(BloomCompactor, t.initBloomCompactor) mm.RegisterModule(BloomCompactorRing, t.initBloomCompactorRing, modules.UserInvisibleModule) + mm.RegisterModule(BloomBuildPlanner, t.initBloomBuildPlanner) mm.RegisterModule(IndexGateway, t.initIndexGateway) mm.RegisterModule(IndexGatewayRing, t.initIndexGatewayRing, modules.UserInvisibleModule) mm.RegisterModule(IndexGatewayInterceptors, t.initIndexGatewayInterceptors, modules.UserInvisibleModule) @@ -686,6 +690,7 @@ func (t *Loki) setupModuleManager() error { IndexGateway: {Server, Store, BloomStore, IndexGatewayRing, IndexGatewayInterceptors, Analytics}, BloomGateway: {Server, BloomStore, Analytics}, BloomCompactor: {Server, BloomStore, BloomCompactorRing, Analytics, Store}, + BloomBuildPlanner: {Server, BloomStore, Analytics, Store}, PatternIngester: {Server, MemberlistKV, Analytics}, PatternRingClient: {Server, MemberlistKV, Analytics}, IngesterQuerier: {Ring}, diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 0280bd514d3c1..542a23510791b 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -27,6 +27,7 @@ import ( "github.com/grafana/dskit/server" "github.com/grafana/dskit/services" "github.com/grafana/dskit/user" + "github.com/grafana/loki/v3/pkg/bloombuild/bloomplanner" gerrors "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -122,6 +123,7 @@ const ( QuerySchedulerRing string = "query-scheduler-ring" BloomCompactor string = "bloom-compactor" BloomCompactorRing string = "bloom-compactor-ring" + BloomBuildPlanner string = "bloom-build-planner" BloomStore string = "bloom-store" All string = "all" Read string = "read" @@ -803,7 +805,7 @@ func (t *Loki) updateConfigForShipperStore() { t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeWriteOnly t.Cfg.StorageConfig.TSDBShipperConfig.IngesterDBRetainPeriod = shipperQuerierIndexUpdateDelay(t.Cfg.StorageConfig.IndexCacheValidity, t.Cfg.StorageConfig.TSDBShipperConfig.ResyncInterval) - case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomCompactor): + case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomBuildPlanner): // We do not want query to do any updates to index t.Cfg.StorageConfig.BoltDBShipperConfig.Mode = indexshipper.ModeReadOnly t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeReadOnly @@ -1553,6 +1555,20 @@ func (t *Loki) initBloomCompactorRing() (services.Service, error) { return t.bloomCompactorRingManager, nil } +func (t *Loki) initBloomBuildPlanner() (services.Service, error) { + if !t.Cfg.BloomBuild.Enabled { + return nil, nil + } + + logger := log.With(util_log.Logger, "component", "bloom-build-planner") + + return bloomplanner.New( + t.Cfg.BloomBuild.Planner, + logger, + prometheus.DefaultRegisterer, + ) +} + func (t *Loki) initQueryScheduler() (services.Service, error) { s, err := scheduler.NewScheduler(t.Cfg.QueryScheduler, t.Overrides, util_log.Logger, t.querySchedulerRingManager, prometheus.DefaultRegisterer, t.Cfg.MetricsNamespace) if err != nil { From b698c229dff13d23f8aded827bc0e63220fead80 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 20 May 2024 10:13:15 +0200 Subject: [PATCH 2/5] Add worker component --- pkg/bloombuild/bloomplanner/metrics.go | 2 +- pkg/bloombuild/bloomplanner/planner.go | 4 ++- pkg/bloombuild/bloomworker/builder.go | 50 ++++++++++++++++++++++++++ pkg/bloombuild/bloomworker/config.go | 21 +++++++++++ pkg/bloombuild/bloomworker/metrics.go | 26 ++++++++++++++ pkg/bloombuild/config.go | 12 +++++-- pkg/loki/loki.go | 2 ++ pkg/loki/modules.go | 20 +++++++++-- 8 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 pkg/bloombuild/bloomworker/builder.go create mode 100644 pkg/bloombuild/bloomworker/config.go create mode 100644 pkg/bloombuild/bloomworker/metrics.go diff --git a/pkg/bloombuild/bloomplanner/metrics.go b/pkg/bloombuild/bloomplanner/metrics.go index 45f99b5b3eef6..2d33197b73492 100644 --- a/pkg/bloombuild/bloomplanner/metrics.go +++ b/pkg/bloombuild/bloomplanner/metrics.go @@ -20,7 +20,7 @@ func NewMetrics(r prometheus.Registerer) *Metrics { Namespace: metricsNamespace, Subsystem: metricsSubsystem, Name: "running", - Help: "Value will be 1 if planner is currently running on this instance", + Help: "Value will be 1 if bloom build planner is currently running on this instance", }), } } diff --git a/pkg/bloombuild/bloomplanner/planner.go b/pkg/bloombuild/bloomplanner/planner.go index c1e38092c6774..766b2dc56716c 100644 --- a/pkg/bloombuild/bloomplanner/planner.go +++ b/pkg/bloombuild/bloomplanner/planner.go @@ -3,9 +3,11 @@ package bloomplanner import ( "context" "github.com/go-kit/log" + "github.com/grafana/dskit/services" - utillog "github.com/grafana/loki/v3/pkg/util/log" "github.com/prometheus/client_golang/prometheus" + + utillog "github.com/grafana/loki/v3/pkg/util/log" ) type Planner struct { diff --git a/pkg/bloombuild/bloomworker/builder.go b/pkg/bloombuild/bloomworker/builder.go new file mode 100644 index 0000000000000..5d3597ab23d1e --- /dev/null +++ b/pkg/bloombuild/bloomworker/builder.go @@ -0,0 +1,50 @@ +package bloomworker + +import ( + "context" + "github.com/go-kit/log" + + "github.com/grafana/dskit/services" + "github.com/prometheus/client_golang/prometheus" + + utillog "github.com/grafana/loki/v3/pkg/util/log" +) + +type Worker struct { + services.Service + + cfg Config + metrics *Metrics + logger log.Logger +} + +func New( + cfg Config, + logger log.Logger, + r prometheus.Registerer, +) (*Worker, error) { + utillog.WarnExperimentalUse("Bloom Builder", logger) + + w := &Worker{ + cfg: cfg, + metrics: NewMetrics(r), + logger: logger, + } + + w.Service = services.NewBasicService(w.starting, w.running, w.stopping) + return w, nil +} + +func (w *Worker) starting(_ context.Context) (err error) { + w.metrics.running.Set(1) + return err +} + +func (w *Worker) stopping(_ error) error { + w.metrics.running.Set(0) + return nil +} + +func (w *Worker) running(_ context.Context) error { + return nil +} diff --git a/pkg/bloombuild/bloomworker/config.go b/pkg/bloombuild/bloomworker/config.go new file mode 100644 index 0000000000000..4284cd1793774 --- /dev/null +++ b/pkg/bloombuild/bloomworker/config.go @@ -0,0 +1,21 @@ +package bloomworker + +import "flag" + +// Config configures the bloom-planner component. +type Config struct { + // TODO: Add config +} + +// RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. +func (cfg *Config) RegisterFlagsWithPrefix(flagsPrefix string, f *flag.FlagSet) { + // TODO: Register flags with flagsPrefix +} + +func (cfg *Config) Validate() error { + return nil +} + +type Limits interface { + // TODO: Add limits +} diff --git a/pkg/bloombuild/bloomworker/metrics.go b/pkg/bloombuild/bloomworker/metrics.go new file mode 100644 index 0000000000000..3bfd6fc80cbb2 --- /dev/null +++ b/pkg/bloombuild/bloomworker/metrics.go @@ -0,0 +1,26 @@ +package bloomworker + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricsNamespace = "loki" + metricsSubsystem = "bloombuilder" +) + +type Metrics struct { + running prometheus.Gauge +} + +func NewMetrics(r prometheus.Registerer) *Metrics { + return &Metrics{ + running: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "running", + Help: "Value will be 1 if the bloom builder is currently running on this instance", + }), + } +} diff --git a/pkg/bloombuild/config.go b/pkg/bloombuild/config.go index 3e1a1babecd0f..f8a850f8cd5b8 100644 --- a/pkg/bloombuild/config.go +++ b/pkg/bloombuild/config.go @@ -3,7 +3,9 @@ package bloombuild import ( "flag" "fmt" - bloomplanner "github.com/grafana/loki/v3/pkg/bloombuild/planner" + + "github.com/grafana/loki/v3/pkg/bloombuild/bloomplanner" + "github.com/grafana/loki/v3/pkg/bloombuild/bloomworker" ) // Config configures the bloom-planner component. @@ -11,12 +13,14 @@ type Config struct { Enabled bool `yaml:"enabled"` Planner bloomplanner.Config `yaml:"planner"` + Worker bloomworker.Config `yaml:"worker"` } // RegisterFlags registers flags for the bloom building configuration. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.Enabled, "bloom-build.enabled", false, "Flag to enable or disable the usage of the bloom-build-planner and bloom-builder components.") - cfg.Planner.RegisterFlagsWithPrefix("bloom-build.enabled.planner", f) + cfg.Planner.RegisterFlagsWithPrefix("bloom-build.planner", f) + cfg.Worker.RegisterFlagsWithPrefix("bloom-build.worker", f) } func (cfg *Config) Validate() error { @@ -28,5 +32,9 @@ func (cfg *Config) Validate() error { return fmt.Errorf("invalid bloom planner configuration: %w", err) } + if err := cfg.Worker.Validate(); err != nil { + return fmt.Errorf("invalid bloom worker configuration: %w", err) + } + return nil } diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index fcbc6d81ca24d..f208d0a4ebb89 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -653,6 +653,7 @@ func (t *Loki) setupModuleManager() error { mm.RegisterModule(BloomCompactor, t.initBloomCompactor) mm.RegisterModule(BloomCompactorRing, t.initBloomCompactorRing, modules.UserInvisibleModule) mm.RegisterModule(BloomBuildPlanner, t.initBloomBuildPlanner) + mm.RegisterModule(BloomBuildWorker, t.initBloomBuildWorker) mm.RegisterModule(IndexGateway, t.initIndexGateway) mm.RegisterModule(IndexGatewayRing, t.initIndexGatewayRing, modules.UserInvisibleModule) mm.RegisterModule(IndexGatewayInterceptors, t.initIndexGatewayInterceptors, modules.UserInvisibleModule) @@ -691,6 +692,7 @@ func (t *Loki) setupModuleManager() error { BloomGateway: {Server, BloomStore, Analytics}, BloomCompactor: {Server, BloomStore, BloomCompactorRing, Analytics, Store}, BloomBuildPlanner: {Server, BloomStore, Analytics, Store}, + BloomBuildWorker: {Server, BloomStore, Analytics, Store}, PatternIngester: {Server, MemberlistKV, Analytics}, PatternRingClient: {Server, MemberlistKV, Analytics}, IngesterQuerier: {Ring}, diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 542a23510791b..6d6a1e538f5e8 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -27,7 +27,6 @@ import ( "github.com/grafana/dskit/server" "github.com/grafana/dskit/services" "github.com/grafana/dskit/user" - "github.com/grafana/loki/v3/pkg/bloombuild/bloomplanner" gerrors "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -39,6 +38,8 @@ import ( "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/analytics" + "github.com/grafana/loki/v3/pkg/bloombuild/bloomplanner" + "github.com/grafana/loki/v3/pkg/bloombuild/bloomworker" "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" compactorclient "github.com/grafana/loki/v3/pkg/compactor/client" @@ -124,6 +125,7 @@ const ( BloomCompactor string = "bloom-compactor" BloomCompactorRing string = "bloom-compactor-ring" BloomBuildPlanner string = "bloom-build-planner" + BloomBuildWorker string = "bloom-build-worker" BloomStore string = "bloom-store" All string = "all" Read string = "read" @@ -805,7 +807,7 @@ func (t *Loki) updateConfigForShipperStore() { t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeWriteOnly t.Cfg.StorageConfig.TSDBShipperConfig.IngesterDBRetainPeriod = shipperQuerierIndexUpdateDelay(t.Cfg.StorageConfig.IndexCacheValidity, t.Cfg.StorageConfig.TSDBShipperConfig.ResyncInterval) - case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomBuildPlanner): + case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomBuildPlanner), t.Cfg.isTarget(BloomBuildWorker): // We do not want query to do any updates to index t.Cfg.StorageConfig.BoltDBShipperConfig.Mode = indexshipper.ModeReadOnly t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeReadOnly @@ -1569,6 +1571,20 @@ func (t *Loki) initBloomBuildPlanner() (services.Service, error) { ) } +func (t *Loki) initBloomBuildWorker() (services.Service, error) { + if !t.Cfg.BloomBuild.Enabled { + return nil, nil + } + + logger := log.With(util_log.Logger, "component", "bloom-build-worker") + + return bloomworker.New( + t.Cfg.BloomBuild.Worker, + logger, + prometheus.DefaultRegisterer, + ) +} + func (t *Loki) initQueryScheduler() (services.Service, error) { s, err := scheduler.NewScheduler(t.Cfg.QueryScheduler, t.Overrides, util_log.Logger, t.querySchedulerRingManager, prometheus.DefaultRegisterer, t.Cfg.MetricsNamespace) if err != nil { From cd1bf6ba68b323d0f836692b423557b73812ea3c Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 20 May 2024 10:21:22 +0200 Subject: [PATCH 3/5] lint issues --- docs/sources/shared/configuration.md | 10 ++++++++++ pkg/bloombuild/bloomplanner/config.go | 2 +- pkg/bloombuild/bloomplanner/planner.go | 2 +- pkg/bloombuild/bloomworker/builder.go | 2 +- pkg/bloombuild/bloomworker/config.go | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index fe57f40daa581..0ac50dfcc14bd 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -326,6 +326,16 @@ pattern_ingester: # merging them as bloom blocks. [bloom_compactor: ] +bloom_build: + # Flag to enable or disable the usage of the bloom-build-planner and + # bloom-builder components. + # CLI flag: -bloom-build.enabled + [enabled: | default = false] + + planner: + + worker: + # Experimental: The bloom_gateway block configures the Loki bloom gateway # server, responsible for serving queries for filtering chunks based on filter # expressions. diff --git a/pkg/bloombuild/bloomplanner/config.go b/pkg/bloombuild/bloomplanner/config.go index b6246461502d9..10e4f5c37a9e9 100644 --- a/pkg/bloombuild/bloomplanner/config.go +++ b/pkg/bloombuild/bloomplanner/config.go @@ -8,7 +8,7 @@ type Config struct { } // RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. -func (cfg *Config) RegisterFlagsWithPrefix(flagsPrefix string, f *flag.FlagSet) { +func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) { // TODO: Register flags with flagsPrefix } diff --git a/pkg/bloombuild/bloomplanner/planner.go b/pkg/bloombuild/bloomplanner/planner.go index 766b2dc56716c..bd7707e0696a0 100644 --- a/pkg/bloombuild/bloomplanner/planner.go +++ b/pkg/bloombuild/bloomplanner/planner.go @@ -2,8 +2,8 @@ package bloomplanner import ( "context" - "github.com/go-kit/log" + "github.com/go-kit/log" "github.com/grafana/dskit/services" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/bloombuild/bloomworker/builder.go b/pkg/bloombuild/bloomworker/builder.go index 5d3597ab23d1e..f1d113ac74675 100644 --- a/pkg/bloombuild/bloomworker/builder.go +++ b/pkg/bloombuild/bloomworker/builder.go @@ -2,8 +2,8 @@ package bloomworker import ( "context" - "github.com/go-kit/log" + "github.com/go-kit/log" "github.com/grafana/dskit/services" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/bloombuild/bloomworker/config.go b/pkg/bloombuild/bloomworker/config.go index 4284cd1793774..5d16bc77c6788 100644 --- a/pkg/bloombuild/bloomworker/config.go +++ b/pkg/bloombuild/bloomworker/config.go @@ -8,7 +8,7 @@ type Config struct { } // RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. -func (cfg *Config) RegisterFlagsWithPrefix(flagsPrefix string, f *flag.FlagSet) { +func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) { // TODO: Register flags with flagsPrefix } From 990a0165bd9b389ce34f3679a4bc205c6c993ec5 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 20 May 2024 11:51:39 +0200 Subject: [PATCH 4/5] rename components --- docs/sources/shared/configuration.md | 6 ++--- .../{bloomworker => builder}/builder.go | 2 +- .../{bloomplanner => builder}/config.go | 4 ++-- .../{bloomworker => builder}/metrics.go | 2 +- pkg/bloombuild/config.go | 17 +++++++------ .../{bloomworker => planner}/config.go | 2 +- .../{bloomplanner => planner}/metrics.go | 4 ++-- .../{bloomplanner => planner}/planner.go | 4 ++-- pkg/loki/loki.go | 8 +++---- pkg/loki/modules.go | 24 +++++++++---------- 10 files changed, 36 insertions(+), 37 deletions(-) rename pkg/bloombuild/{bloomworker => builder}/builder.go (97%) rename pkg/bloombuild/{bloomplanner => builder}/config.go (83%) rename pkg/bloombuild/{bloomworker => builder}/metrics.go (96%) rename pkg/bloombuild/{bloomworker => planner}/config.go (95%) rename pkg/bloombuild/{bloomplanner => planner}/metrics.go (80%) rename pkg/bloombuild/{bloomplanner => planner}/planner.go (90%) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 0ac50dfcc14bd..f59f6501c94e5 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -327,14 +327,14 @@ pattern_ingester: [bloom_compactor: ] bloom_build: - # Flag to enable or disable the usage of the bloom-build-planner and - # bloom-builder components. + # Flag to enable or disable the usage of the bloom-planner and bloom-builder + # components. # CLI flag: -bloom-build.enabled [enabled: | default = false] planner: - worker: + builder: # Experimental: The bloom_gateway block configures the Loki bloom gateway # server, responsible for serving queries for filtering chunks based on filter diff --git a/pkg/bloombuild/bloomworker/builder.go b/pkg/bloombuild/builder/builder.go similarity index 97% rename from pkg/bloombuild/bloomworker/builder.go rename to pkg/bloombuild/builder/builder.go index f1d113ac74675..098e7d6d83f00 100644 --- a/pkg/bloombuild/bloomworker/builder.go +++ b/pkg/bloombuild/builder/builder.go @@ -1,4 +1,4 @@ -package bloomworker +package builder import ( "context" diff --git a/pkg/bloombuild/bloomplanner/config.go b/pkg/bloombuild/builder/config.go similarity index 83% rename from pkg/bloombuild/bloomplanner/config.go rename to pkg/bloombuild/builder/config.go index 10e4f5c37a9e9..ac282ccf95ebb 100644 --- a/pkg/bloombuild/bloomplanner/config.go +++ b/pkg/bloombuild/builder/config.go @@ -1,8 +1,8 @@ -package bloomplanner +package builder import "flag" -// Config configures the bloom-planner component. +// Config configures the bloom-builder component. type Config struct { // TODO: Add config } diff --git a/pkg/bloombuild/bloomworker/metrics.go b/pkg/bloombuild/builder/metrics.go similarity index 96% rename from pkg/bloombuild/bloomworker/metrics.go rename to pkg/bloombuild/builder/metrics.go index 3bfd6fc80cbb2..e8f46fa025080 100644 --- a/pkg/bloombuild/bloomworker/metrics.go +++ b/pkg/bloombuild/builder/metrics.go @@ -1,4 +1,4 @@ -package bloomworker +package builder import ( "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/bloombuild/config.go b/pkg/bloombuild/config.go index f8a850f8cd5b8..a74adb51fc221 100644 --- a/pkg/bloombuild/config.go +++ b/pkg/bloombuild/config.go @@ -3,24 +3,23 @@ package bloombuild import ( "flag" "fmt" - - "github.com/grafana/loki/v3/pkg/bloombuild/bloomplanner" - "github.com/grafana/loki/v3/pkg/bloombuild/bloomworker" + "github.com/grafana/loki/v3/pkg/bloombuild/builder" + "github.com/grafana/loki/v3/pkg/bloombuild/planner" ) // Config configures the bloom-planner component. type Config struct { Enabled bool `yaml:"enabled"` - Planner bloomplanner.Config `yaml:"planner"` - Worker bloomworker.Config `yaml:"worker"` + Planner planner.Config `yaml:"planner"` + Builder builder.Config `yaml:"builder"` } // RegisterFlags registers flags for the bloom building configuration. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.BoolVar(&cfg.Enabled, "bloom-build.enabled", false, "Flag to enable or disable the usage of the bloom-build-planner and bloom-builder components.") + f.BoolVar(&cfg.Enabled, "bloom-build.enabled", false, "Flag to enable or disable the usage of the bloom-planner and bloom-builder components.") cfg.Planner.RegisterFlagsWithPrefix("bloom-build.planner", f) - cfg.Worker.RegisterFlagsWithPrefix("bloom-build.worker", f) + cfg.Builder.RegisterFlagsWithPrefix("bloom-build.builder", f) } func (cfg *Config) Validate() error { @@ -32,8 +31,8 @@ func (cfg *Config) Validate() error { return fmt.Errorf("invalid bloom planner configuration: %w", err) } - if err := cfg.Worker.Validate(); err != nil { - return fmt.Errorf("invalid bloom worker configuration: %w", err) + if err := cfg.Builder.Validate(); err != nil { + return fmt.Errorf("invalid bloom builder configuration: %w", err) } return nil diff --git a/pkg/bloombuild/bloomworker/config.go b/pkg/bloombuild/planner/config.go similarity index 95% rename from pkg/bloombuild/bloomworker/config.go rename to pkg/bloombuild/planner/config.go index 5d16bc77c6788..dd8cb315d9345 100644 --- a/pkg/bloombuild/bloomworker/config.go +++ b/pkg/bloombuild/planner/config.go @@ -1,4 +1,4 @@ -package bloomworker +package planner import "flag" diff --git a/pkg/bloombuild/bloomplanner/metrics.go b/pkg/bloombuild/planner/metrics.go similarity index 80% rename from pkg/bloombuild/bloomplanner/metrics.go rename to pkg/bloombuild/planner/metrics.go index 2d33197b73492..e9a9035e14df0 100644 --- a/pkg/bloombuild/bloomplanner/metrics.go +++ b/pkg/bloombuild/planner/metrics.go @@ -1,4 +1,4 @@ -package bloomplanner +package planner import ( "github.com/prometheus/client_golang/prometheus" @@ -20,7 +20,7 @@ func NewMetrics(r prometheus.Registerer) *Metrics { Namespace: metricsNamespace, Subsystem: metricsSubsystem, Name: "running", - Help: "Value will be 1 if bloom build planner is currently running on this instance", + Help: "Value will be 1 if bloom planner is currently running on this instance", }), } } diff --git a/pkg/bloombuild/bloomplanner/planner.go b/pkg/bloombuild/planner/planner.go similarity index 90% rename from pkg/bloombuild/bloomplanner/planner.go rename to pkg/bloombuild/planner/planner.go index bd7707e0696a0..7732d180b0bb8 100644 --- a/pkg/bloombuild/bloomplanner/planner.go +++ b/pkg/bloombuild/planner/planner.go @@ -1,4 +1,4 @@ -package bloomplanner +package planner import ( "context" @@ -23,7 +23,7 @@ func New( logger log.Logger, r prometheus.Registerer, ) (*Planner, error) { - utillog.WarnExperimentalUse("Bloom Build Planner", logger) + utillog.WarnExperimentalUse("Bloom Planner", logger) p := &Planner{ cfg: cfg, diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index f208d0a4ebb89..9446b351aab82 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -652,8 +652,8 @@ func (t *Loki) setupModuleManager() error { mm.RegisterModule(BloomStore, t.initBloomStore) mm.RegisterModule(BloomCompactor, t.initBloomCompactor) mm.RegisterModule(BloomCompactorRing, t.initBloomCompactorRing, modules.UserInvisibleModule) - mm.RegisterModule(BloomBuildPlanner, t.initBloomBuildPlanner) - mm.RegisterModule(BloomBuildWorker, t.initBloomBuildWorker) + mm.RegisterModule(BloomPlanner, t.initBloomPlanner) + mm.RegisterModule(BloomBuilder, t.initBloomBuilder) mm.RegisterModule(IndexGateway, t.initIndexGateway) mm.RegisterModule(IndexGatewayRing, t.initIndexGatewayRing, modules.UserInvisibleModule) mm.RegisterModule(IndexGatewayInterceptors, t.initIndexGatewayInterceptors, modules.UserInvisibleModule) @@ -691,8 +691,8 @@ func (t *Loki) setupModuleManager() error { IndexGateway: {Server, Store, BloomStore, IndexGatewayRing, IndexGatewayInterceptors, Analytics}, BloomGateway: {Server, BloomStore, Analytics}, BloomCompactor: {Server, BloomStore, BloomCompactorRing, Analytics, Store}, - BloomBuildPlanner: {Server, BloomStore, Analytics, Store}, - BloomBuildWorker: {Server, BloomStore, Analytics, Store}, + BloomPlanner: {Server, BloomStore, Analytics, Store}, + BloomBuilder: {Server, BloomStore, Analytics, Store}, PatternIngester: {Server, MemberlistKV, Analytics}, PatternRingClient: {Server, MemberlistKV, Analytics}, IngesterQuerier: {Ring}, diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 6d6a1e538f5e8..a563e80f789fe 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -38,8 +38,8 @@ import ( "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/analytics" - "github.com/grafana/loki/v3/pkg/bloombuild/bloomplanner" - "github.com/grafana/loki/v3/pkg/bloombuild/bloomworker" + "github.com/grafana/loki/v3/pkg/bloombuild/builder" + "github.com/grafana/loki/v3/pkg/bloombuild/planner" "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" compactorclient "github.com/grafana/loki/v3/pkg/compactor/client" @@ -124,8 +124,8 @@ const ( QuerySchedulerRing string = "query-scheduler-ring" BloomCompactor string = "bloom-compactor" BloomCompactorRing string = "bloom-compactor-ring" - BloomBuildPlanner string = "bloom-build-planner" - BloomBuildWorker string = "bloom-build-worker" + BloomPlanner string = "bloom-planner" + BloomBuilder string = "bloom-builder" BloomStore string = "bloom-store" All string = "all" Read string = "read" @@ -807,7 +807,7 @@ func (t *Loki) updateConfigForShipperStore() { t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeWriteOnly t.Cfg.StorageConfig.TSDBShipperConfig.IngesterDBRetainPeriod = shipperQuerierIndexUpdateDelay(t.Cfg.StorageConfig.IndexCacheValidity, t.Cfg.StorageConfig.TSDBShipperConfig.ResyncInterval) - case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomBuildPlanner), t.Cfg.isTarget(BloomBuildWorker): + case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomCompactor), t.Cfg.isTarget(BloomPlanner), t.Cfg.isTarget(BloomBuilder): // We do not want query to do any updates to index t.Cfg.StorageConfig.BoltDBShipperConfig.Mode = indexshipper.ModeReadOnly t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeReadOnly @@ -1557,29 +1557,29 @@ func (t *Loki) initBloomCompactorRing() (services.Service, error) { return t.bloomCompactorRingManager, nil } -func (t *Loki) initBloomBuildPlanner() (services.Service, error) { +func (t *Loki) initBloomPlanner() (services.Service, error) { if !t.Cfg.BloomBuild.Enabled { return nil, nil } - logger := log.With(util_log.Logger, "component", "bloom-build-planner") + logger := log.With(util_log.Logger, "component", "bloom-planner") - return bloomplanner.New( + return planner.New( t.Cfg.BloomBuild.Planner, logger, prometheus.DefaultRegisterer, ) } -func (t *Loki) initBloomBuildWorker() (services.Service, error) { +func (t *Loki) initBloomBuilder() (services.Service, error) { if !t.Cfg.BloomBuild.Enabled { return nil, nil } - logger := log.With(util_log.Logger, "component", "bloom-build-worker") + logger := log.With(util_log.Logger, "component", "bloom-worker") - return bloomworker.New( - t.Cfg.BloomBuild.Worker, + return builder.New( + t.Cfg.BloomBuild.Builder, logger, prometheus.DefaultRegisterer, ) From 2d883171e0b5f7530c50eda437f96dcf8f2757f3 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 20 May 2024 12:02:30 +0200 Subject: [PATCH 5/5] lint --- pkg/bloombuild/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/bloombuild/config.go b/pkg/bloombuild/config.go index a74adb51fc221..c69c605607f5a 100644 --- a/pkg/bloombuild/config.go +++ b/pkg/bloombuild/config.go @@ -3,6 +3,7 @@ package bloombuild import ( "flag" "fmt" + "github.com/grafana/loki/v3/pkg/bloombuild/builder" "github.com/grafana/loki/v3/pkg/bloombuild/planner" )