feat: Add settings for cpu/mutex/block profiling options (#13278)

The HTTP endpoints for scraping CPU, mutex and block profiles are already exposed by the server, however, block and mutex profiling must be explicitly enabled by setting a non-zero value for `SetBlockProfileRate` and `SetMutexProfileFraction` respectively.

For more information about the available options consult the Go documentation:

* https://pkg.go.dev/runtime#SetBlockProfileRate
* https://pkg.go.dev/runtime#SetMutexProfileFraction
* https://pkg.go.dev/runtime#SetCPUProfileRate

Signed-off-by: Christian Haudum <[email protected]>

Author: chaudum

cmd/loki/main.go

@@ -106,6 +106,8 @@ func main() {
 		}()
 	}
 
+	setProfilingOptions(config.Profiling)
+
 	// Allocate a block of memory to reduce the frequency of garbage collection.
 	// The larger the ballast, the lower the garbage collection frequency.
 	// https://github.com/grafana/loki/issues/781
@@ -127,3 +129,15 @@ func main() {
 	err = t.Run(loki.RunOpts{StartTime: startTime})
 	util_log.CheckFatal("running loki", err, util_log.Logger)
 }
+
+func setProfilingOptions(cfg loki.ProfilingConfig) {
+	if cfg.BlockProfileRate > 0 {
+		runtime.SetBlockProfileRate(cfg.BlockProfileRate)
+	}
+	if cfg.CPUProfileRate > 0 {
+		runtime.SetCPUProfileRate(cfg.CPUProfileRate)
+	}
+	if cfg.MutexProfileFraction > 0 {
+		runtime.SetMutexProfileFraction(cfg.MutexProfileFraction)
+	}
+}

docs/sources/shared/configuration.md

@@ -567,6 +567,9 @@ compactor_grpc_client:
 # Configuration for analytics.
 [analytics: <analytics>]
 
+# Configuration for profiling options.
+[profiling: <profiling>]
+
 # Common configuration to be shared between multiple modules. If a more specific
 # configuration is given in other sections, the related configuration within
 # this section will be ignored.
@@ -3850,6 +3853,24 @@ chunks:
 [row_shards: <int> | default = 16]
 ```
 
+### profiling
+
+Configuration for `profiling` options.
+
+```yaml
+# Sets the value for runtime.SetBlockProfilingRate
+# CLI flag: -profiling.block-profile-rate
+[block_profile_rate: <int> | default = 0]
+
+# Sets the value for runtime.SetCPUProfileRate
+# CLI flag: -profiling.cpu-profile-rate
+[cpu_profile_rate: <int> | default = 0]
+
+# Sets the value for runtime.SetMutexProfileFraction
+# CLI flag: -profiling.mutex-profile-fraction
+[mutex_profile_fraction: <int> | default = 0]
+```
+
 ### querier
 
 Configures the `querier`. Only appropriate when running all modules or just the querier.

pkg/loki/loki.go

@@ -108,6 +108,7 @@ type Config struct {
 	OperationalConfig runtime.Config       `yaml:"operational_config,omitempty"`
 	Tracing           tracing.Config       `yaml:"tracing"`
 	Analytics         analytics.Config     `yaml:"analytics"`
+	Profiling         ProfilingConfig      `yaml:"profiling,omitempty"`
 
 	LegacyReadTarget bool `yaml:"legacy_read_target,omitempty" doc:"hidden|deprecated"`
 
@@ -179,6 +180,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
 	c.QueryScheduler.RegisterFlags(f)
 	c.Analytics.RegisterFlags(f)
 	c.OperationalConfig.RegisterFlags(f)
+	c.Profiling.RegisterFlags(f)
 }
 
 func (c *Config) registerServerFlagsWithChangedDefaultValues(fs *flag.FlagSet) {

pkg/loki/profiling_config.go

@@ -0,0 +1,21 @@
+package loki
+
+import "flag"
+
+type ProfilingConfig struct {
+	BlockProfileRate     int `yaml:"block_profile_rate"`
+	CPUProfileRate       int `yaml:"cpu_profile_rate"`
+	MutexProfileFraction int `yaml:"mutex_profile_fraction"`
+}
+
+// RegisterFlags registers flag.
+func (c *ProfilingConfig) RegisterFlags(f *flag.FlagSet) {
+	c.RegisterFlagsWithPrefix("profiling.", f)
+}
+
+// RegisterFlagsWithPrefix registers flag with a common prefix.
+func (c *ProfilingConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.IntVar(&c.BlockProfileRate, prefix+"block-profile-rate", 0, "Sets the value for runtime.SetBlockProfilingRate")
+	f.IntVar(&c.CPUProfileRate, prefix+"cpu-profile-rate", 0, "Sets the value for runtime.SetCPUProfileRate")
+	f.IntVar(&c.MutexProfileFraction, prefix+"mutex-profile-fraction", 0, "Sets the value for runtime.SetMutexProfileFraction")
+}

tools/doc-generator/parse/root_blocks.go

@@ -23,6 +23,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/ingester"
 	ingester_client "github.com/grafana/loki/v3/pkg/ingester/client"
 	"github.com/grafana/loki/v3/pkg/loghttp/push"
+	"github.com/grafana/loki/v3/pkg/loki"
 	"github.com/grafana/loki/v3/pkg/loki/common"
 	frontend "github.com/grafana/loki/v3/pkg/lokifrontend"
 	"github.com/grafana/loki/v3/pkg/querier"
@@ -168,6 +169,11 @@ var (
 			StructType: []reflect.Type{reflect.TypeOf(analytics.Config{})},
 			Desc:       "Configuration for analytics.",
 		},
+		{
+			Name:       "profiling",
+			StructType: []reflect.Type{reflect.TypeOf(loki.ProfilingConfig{})},
+			Desc:       "Configuration for profiling options.",
+		},
 
 		{
 			Name:       "common",