From 672ca4ed1e5e4546d1935abb72294c3a9ba07315 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 26 Sep 2018 12:06:38 -0400 Subject: [PATCH 1/3] Allow to pass config overrides via the Settings Struct Sometime a custom beat that get executed want to override system defaults instead of relying on code defined by libbeat. This is the case with beatless, the queue has limits and flush values that make sense when the beat is run on the edge but not on on AWS lambda. Note that settings is passed via liberal common.Config no type checking is done. I went that route because many parts of beats doesn't expose the config struct outside of the package. This is similar to explicitely defining them in the yaml. --- libbeat/cmd/instance/beat.go | 27 +++++++++++++++++++++++---- libbeat/cmd/instance/settings.go | 12 +++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index fc7e3f06270c..aebb6be1d6cc 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -229,8 +229,8 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) { return &Beat{Beat: b}, nil } -// init does initialization of things common to all actions (read confs, flags) -func (b *Beat) Init() error { +// InitWithSettings does initialization of things common to all actions (read confs, flags) +func (b *Beat) InitWithSettings(settings Settings) error { err := b.handleFlags() if err != nil { return err @@ -240,13 +240,20 @@ func (b *Beat) Init() error { return err } - if err := b.configure(); err != nil { + if err := b.configure(settings); err != nil { return err } return nil } +// Init does initialization of things common to all actions (read confs, flags) +// +// Deprecated: use InitWithSettings +func (b *Beat) Init() error { + return b.InitWithSettings(Settings{}) +} + // BeatConfig returns config section for this beat func (b *Beat) BeatConfig() (*common.Config, error) { configName := strings.ToLower(b.Info.Beat) @@ -504,7 +511,7 @@ func (b *Beat) handleFlags() error { // config reads the configuration file from disk, parses the common options // defined in BeatConfig, initializes logging, and set GOMAXPROCS if defined // in the config. Lastly it invokes the Config method implemented by the beat. -func (b *Beat) configure() error { +func (b *Beat) configure(settings Settings) error { var err error cfg, err := cfgfile.Load("") @@ -512,6 +519,18 @@ func (b *Beat) configure() error { return fmt.Errorf("error loading config file: %v", err) } + // Overrides config defaults for a specific beat. + if settings.ConfigOverrides != nil { + c, _ := common.NewConfigFrom(map[string]interface{}{}) + if err := c.Merge(settings.ConfigOverrides); err != nil { + return err + } + if err := c.Merge(cfg); err != nil { + return err + } + cfg = c + } + // We have to initialize the keystore before any unpack or merging the cloud // options. keystoreCfg, _ := cfg.Child("keystore", -1) diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index 5c01c33e22f9..09c1180a56f8 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -20,14 +20,16 @@ package instance import ( "github.com/spf13/pflag" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/monitoring/report" ) // Settings contains basic settings for any beat to pass into GenRootCmd type Settings struct { - Name string - IndexPrefix string - Version string - Monitoring report.Settings - RunFlags *pflag.FlagSet + Name string + IndexPrefix string + Version string + Monitoring report.Settings + RunFlags *pflag.FlagSet + ConfigOverrides *common.Config } From 8045d319fea188b4fa42f89b820572f03c891d1e Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 26 Sep 2018 16:24:52 -0400 Subject: [PATCH 2/3] make sure the settings are pass all around correctly --- libbeat/cfgfile/cfgfile.go | 37 ++++++++++++++++++------------------ libbeat/cmd/instance/beat.go | 16 ++-------------- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/libbeat/cfgfile/cfgfile.go b/libbeat/cfgfile/cfgfile.go index 13407864c0b7..2ead889e8c88 100644 --- a/libbeat/cfgfile/cfgfile.go +++ b/libbeat/cfgfile/cfgfile.go @@ -37,7 +37,7 @@ var ( testConfig = flag.Bool("configtest", false, "Test configuration and exit.") // Additional default settings, that must be available for variable expansion - defaults = mustNewConfigFrom(map[string]interface{}{ + defaults = common.MustNewConfigFrom(map[string]interface{}{ "path": map[string]interface{}{ "home": ".", // to be initialized by beat "config": "${path.home}", @@ -63,14 +63,6 @@ func init() { makePathFlag("path.logs", "Logs path") } -func mustNewConfigFrom(from interface{}) *common.Config { - cfg, err := common.NewConfigFrom(from) - if err != nil { - panic(err) - } - return cfg -} - // ChangeDefaultCfgfileFlag replaces the value and default value for the `-c` // flag so that it reflects the beat name. func ChangeDefaultCfgfileFlag(beatName string) error { @@ -105,7 +97,7 @@ func HandleFlags() error { // structure. If path is empty this method reads from the configuration // file specified by the '-c' command line flag. func Read(out interface{}, path string) error { - config, err := Load(path) + config, err := Load(path, nil) if err != nil { return err } @@ -116,7 +108,7 @@ func Read(out interface{}, path string) error { // Load reads the configuration from a YAML file structure. If path is empty // this method reads from the configuration file specified by the '-c' command // line flag. -func Load(path string) (*common.Config, error) { +func Load(path string, beatOverrides *common.Config) (*common.Config, error) { var config *common.Config var err error @@ -142,13 +134,22 @@ func Load(path string) (*common.Config, error) { return nil, err } - config, err = common.MergeConfigs( - defaults, - config, - overwrites, - ) - if err != nil { - return nil, err + if beatOverrides != nil { + config, err = common.MergeConfigs( + defaults, + beatOverrides, + config, + overwrites, + ) + if err != nil { + return nil, err + } + } else { + config, err = common.MergeConfigs( + defaults, + config, + overwrites, + ) } config.PrintDebugf("Complete configuration loaded:") diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index aebb6be1d6cc..548e26514110 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -330,7 +330,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { defer logp.Sync() defer logp.Info("%s stopped.", b.Info.Beat) - err := b.Init() + err := b.InitWithSettings(settings) if err != nil { return err } @@ -514,23 +514,11 @@ func (b *Beat) handleFlags() error { func (b *Beat) configure(settings Settings) error { var err error - cfg, err := cfgfile.Load("") + cfg, err := cfgfile.Load("", settings.ConfigOverrides) if err != nil { return fmt.Errorf("error loading config file: %v", err) } - // Overrides config defaults for a specific beat. - if settings.ConfigOverrides != nil { - c, _ := common.NewConfigFrom(map[string]interface{}{}) - if err := c.Merge(settings.ConfigOverrides); err != nil { - return err - } - if err := c.Merge(cfg); err != nil { - return err - } - cfg = c - } - // We have to initialize the keystore before any unpack or merging the cloud // options. keystoreCfg, _ := cfg.Child("keystore", -1) From 5103c8eb8c6b33515f1059a09a5c29fdc88a0ce6 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 26 Sep 2018 16:26:12 -0400 Subject: [PATCH 3/3] added developper changelog --- CHANGELOG-developer.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index 1a4774a20874..67ae4a28ff77 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -51,3 +51,4 @@ The list below covers the major changes between 6.3.0 and master only. - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850] - Set current year in generator templates. {pull}8396[8396] +- You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]