From 01eaf981e5ced94a6ddaba9e97ee04afe7c4f347 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 1 Nov 2019 16:02:31 +0200 Subject: [PATCH 1/8] Move lightmodules functionality to OSS Signed-off-by: chrismark --- metricbeat/beater/metricbeat.go | 10 ++++ metricbeat/cmd/root.go | 1 + .../mb/lightmodules.go | 34 +++++++---- .../mb/lightmodules_test.go | 58 +++++++++++-------- .../testdata/lightmodules/broken/module.yml | 3 + .../testdata/lightmodules/empty/.placeholder | 0 .../service/metricset/manifest.yml | 6 ++ .../testdata/lightmodules/service/module.yml | 4 ++ .../service/nondefault/manifest.yml | 4 ++ x-pack/metricbeat/beater/metricbeat.go | 20 ------- x-pack/metricbeat/cmd/root.go | 5 +- 11 files changed, 88 insertions(+), 57 deletions(-) rename {x-pack/metricbeat => metricbeat}/mb/lightmodules.go (82%) rename {x-pack/metricbeat => metricbeat}/mb/lightmodules_test.go (81%) create mode 100644 metricbeat/mb/testdata/lightmodules/broken/module.yml create mode 100644 metricbeat/mb/testdata/lightmodules/empty/.placeholder create mode 100644 metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml create mode 100644 metricbeat/mb/testdata/lightmodules/service/module.yml create mode 100644 metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml delete mode 100644 x-pack/metricbeat/beater/metricbeat.go diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index eb8e61f70abd..993401e46f4f 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -18,6 +18,7 @@ package beater import ( + "github.com/elastic/beats/libbeat/paths" "sync" "github.com/elastic/beats/libbeat/common/reload" @@ -69,6 +70,14 @@ func WithModuleOptions(options ...module.Option) Option { } } +// WithLightModules enables light modules support +func WithLightModules() Option { + return func(*Metricbeat) { + path := paths.Resolve(paths.Home, "module") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource(path)) + } +} + // Creator returns a beat.Creator for instantiating a new instance of the // Metricbeat framework with the given options. func Creator(options ...Option) beat.Creator { @@ -90,6 +99,7 @@ func Creator(options ...Option) beat.Creator { // ) func DefaultCreator() beat.Creator { return Creator( + WithLightModules(), WithModuleOptions( module.WithMetricSetInfo(), module.WithServiceName(), diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index 9c0d24cc932c..4fe2bf966f6c 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -44,6 +44,7 @@ var ( // been disabled to workaround the fact that Modules() will return // the static modules (not the dynamic ones) with a start delay. testModulesCreator = beater.Creator( + beater.WithLightModules(), beater.WithModuleOptions( module.WithMetricSetInfo(), module.WithMaxStartDelay(0), diff --git a/x-pack/metricbeat/mb/lightmodules.go b/metricbeat/mb/lightmodules.go similarity index 82% rename from x-pack/metricbeat/mb/lightmodules.go rename to metricbeat/mb/lightmodules.go index d01d887a72d0..f39e8152ca91 100644 --- a/x-pack/metricbeat/mb/lightmodules.go +++ b/metricbeat/mb/lightmodules.go @@ -1,6 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package mb @@ -15,7 +28,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" ) const ( @@ -105,15 +117,15 @@ func (s *LightModulesSource) HasMetricSet(moduleName, metricSetName string) bool } // MetricSetRegistration obtains a registration for a light metric set -func (s *LightModulesSource) MetricSetRegistration(register *mb.Register, moduleName, metricSetName string) (mb.MetricSetRegistration, error) { +func (s *LightModulesSource) MetricSetRegistration(register *Register, moduleName, metricSetName string) (MetricSetRegistration, error) { lightModule, err := s.loadModule(moduleName) if err != nil { - return mb.MetricSetRegistration{}, errors.Wrapf(err, "failed to load module '%s'", moduleName) + return MetricSetRegistration{}, errors.Wrapf(err, "failed to load module '%s'", moduleName) } ms, found := lightModule.MetricSets[metricSetName] if !found { - return mb.MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' not found", moduleName, metricSetName) + return MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' not found", moduleName, metricSetName) } return ms.Registration(register) @@ -141,7 +153,7 @@ type lightModuleConfig struct { // LightModule contains the definition of a light module type LightModule struct { Name string - MetricSets map[string]mb.LightMetricSet + MetricSets map[string]LightMetricSet } func (s *LightModulesSource) loadModule(moduleName string) (*LightModule, error) { @@ -186,8 +198,8 @@ func (s *LightModulesSource) loadModuleConfig(modulePath string) (*lightModuleCo return &moduleConfig, nil } -func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, metricSetNames []string) (map[string]mb.LightMetricSet, error) { - metricSets := make(map[string]mb.LightMetricSet) +func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, metricSetNames []string) (map[string]LightMetricSet, error) { + metricSets := make(map[string]LightMetricSet) for _, metricSet := range metricSetNames { manifestPath := filepath.Join(moduleDirPath, metricSet, manifestYML) @@ -203,7 +215,7 @@ func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, me return metricSets, nil } -func (s *LightModulesSource) loadMetricSetConfig(manifestPath string) (ms mb.LightMetricSet, err error) { +func (s *LightModulesSource) loadMetricSetConfig(manifestPath string) (ms LightMetricSet, err error) { config, err := common.LoadFile(manifestPath) if err != nil { return ms, errors.Wrapf(err, "failed to load metricset manifest from '%s'", manifestPath) diff --git a/x-pack/metricbeat/mb/lightmodules_test.go b/metricbeat/mb/lightmodules_test.go similarity index 81% rename from x-pack/metricbeat/mb/lightmodules_test.go rename to metricbeat/mb/lightmodules_test.go index 71ea2f550a7c..e24c5d847eb5 100644 --- a/x-pack/metricbeat/mb/lightmodules_test.go +++ b/metricbeat/mb/lightmodules_test.go @@ -1,6 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. // +build !integration @@ -15,7 +28,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" ) // TestLightModulesAsModuleSource checks that registry correctly lists @@ -27,7 +39,7 @@ func TestLightModulesAsModuleSource(t *testing.T) { name string module string isDefault bool - hostParser mb.HostParser + hostParser HostParser } cases := map[string]struct { @@ -77,19 +89,19 @@ func TestLightModulesAsModuleSource(t *testing.T) { }, } - fakeMetricSetFactory := func(base mb.BaseMetricSet) (mb.MetricSet, error) { + fakeMetricSetFactory := func(base BaseMetricSet) (MetricSet, error) { return &base, nil } - newRegistry := func(metricSets []testMetricSet) *mb.Register { - r := mb.NewRegister() + newRegistry := func(metricSets []testMetricSet) *Register { + r := NewRegister() for _, m := range metricSets { - opts := []mb.MetricSetOption{} + opts := []MetricSetOption{} if m.isDefault { - opts = append(opts, mb.DefaultMetricSet()) + opts = append(opts, DefaultMetricSet()) } if m.hostParser != nil { - opts = append(opts, mb.WithHostParser(m.hostParser)) + opts = append(opts, WithHostParser(m.hostParser)) } r.MustAddMetricSet(m.module, m.name, fakeMetricSetFactory, opts...) } @@ -174,7 +186,7 @@ func TestNewModuleFromConfig(t *testing.T) { config common.MapStr err bool expectedOption string - expectedQuery mb.QueryParams + expectedQuery QueryParams expectedPeriod time.Duration }{ "normal module": { @@ -196,7 +208,7 @@ func TestNewModuleFromConfig(t *testing.T) { "light module with query": { config: common.MapStr{"module": "service", "query": common.MapStr{"param": "foo"}}, expectedOption: "test", - expectedQuery: mb.QueryParams{"param": "foo"}, + expectedQuery: QueryParams{"param": "foo"}, }, "light module with custom period": { config: common.MapStr{"module": "service", "period": "42s"}, @@ -217,7 +229,7 @@ func TestNewModuleFromConfig(t *testing.T) { }, } - r := mb.NewRegister() + r := NewRegister() r.MustAddMetricSet("foo", "bar", newMetricSetWithOption) r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) @@ -226,7 +238,7 @@ func TestNewModuleFromConfig(t *testing.T) { config, err := common.NewConfigFrom(c.config) require.NoError(t, err) - module, metricSets, err := mb.NewModule(config, r) + module, metricSets, err := NewModule(config, r) if c.err { assert.Error(t, err) return @@ -248,7 +260,7 @@ func TestNewModuleFromConfig(t *testing.T) { assert.Equal(t, c.expectedQuery, ms.Module().Config().Query) expectedPeriod := c.expectedPeriod if expectedPeriod == 0 { - expectedPeriod = mb.DefaultModuleConfig().Period + expectedPeriod = DefaultModuleConfig().Period } assert.Equal(t, expectedPeriod, ms.Module().Config().Period) }) @@ -260,31 +272,31 @@ func TestNewModuleFromConfig(t *testing.T) { func TestNewModulesCallModuleFactory(t *testing.T) { logp.TestingSetup() - r := mb.NewRegister() + r := NewRegister() r.MustAddMetricSet("foo", "bar", newMetricSetWithOption) r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) called := false - r.AddModule("foo", func(base mb.BaseModule) (mb.Module, error) { + r.AddModule("foo", func(base BaseModule) (Module, error) { called = true - return mb.DefaultModuleFactory(base) + return DefaultModuleFactory(base) }) config, err := common.NewConfigFrom(common.MapStr{"module": "service"}) require.NoError(t, err) - _, _, err = mb.NewModule(config, r) + _, _, err = NewModule(config, r) assert.NoError(t, err) assert.True(t, called, "module factory must be called if registered") } type metricSetWithOption struct { - mb.BaseMetricSet + BaseMetricSet Option string } -func newMetricSetWithOption(base mb.BaseMetricSet) (mb.MetricSet, error) { +func newMetricSetWithOption(base BaseMetricSet) (MetricSet, error) { config := struct { Option string `config:"option"` }{ @@ -301,4 +313,4 @@ func newMetricSetWithOption(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } -func (*metricSetWithOption) Fetch(mb.ReporterV2) error { return nil } +func (*metricSetWithOption) Fetch(ReporterV2) error { return nil } diff --git a/metricbeat/mb/testdata/lightmodules/broken/module.yml b/metricbeat/mb/testdata/lightmodules/broken/module.yml new file mode 100644 index 000000000000..a4b4e73c3685 --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/broken/module.yml @@ -0,0 +1,3 @@ +name: broken +metricsets: +- notexists diff --git a/metricbeat/mb/testdata/lightmodules/empty/.placeholder b/metricbeat/mb/testdata/lightmodules/empty/.placeholder new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml b/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml new file mode 100644 index 000000000000..5291cac44e68 --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml @@ -0,0 +1,6 @@ +default: true +input: + module: foo + metricset: bar + defaults: + option: test diff --git a/metricbeat/mb/testdata/lightmodules/service/module.yml b/metricbeat/mb/testdata/lightmodules/service/module.yml new file mode 100644 index 000000000000..7d037081eeb0 --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/service/module.yml @@ -0,0 +1,4 @@ +name: service +metricsets: +- metricset +- nondefault diff --git a/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml b/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml new file mode 100644 index 000000000000..f4dc493d29c0 --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml @@ -0,0 +1,4 @@ +default: false +input: + module: foo + metricset: baz diff --git a/x-pack/metricbeat/beater/metricbeat.go b/x-pack/metricbeat/beater/metricbeat.go deleted file mode 100644 index 09069aa2f269..000000000000 --- a/x-pack/metricbeat/beater/metricbeat.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package beater - -import ( - "github.com/elastic/beats/libbeat/paths" - "github.com/elastic/beats/metricbeat/beater" - "github.com/elastic/beats/metricbeat/mb" - xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" -) - -// WithLightModules enables light modules support -func WithLightModules() beater.Option { - return func(*beater.Metricbeat) { - path := paths.Resolve(paths.Home, "module") - mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource(path)) - } -} diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 684ce920f24d..5c3451e03209 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -16,7 +16,6 @@ import ( "github.com/elastic/beats/metricbeat/cmd/test" "github.com/elastic/beats/metricbeat/mb/module" xpackcmd "github.com/elastic/beats/x-pack/libbeat/cmd" - xpackbeater "github.com/elastic/beats/x-pack/metricbeat/beater" // Register the includes. _ "github.com/elastic/beats/x-pack/metricbeat/include" @@ -34,7 +33,7 @@ var RootCmd *cmd.BeatsRootCmd var ( rootCreator = beater.Creator( - xpackbeater.WithLightModules(), + beater.WithLightModules(), beater.WithModuleOptions( module.WithMetricSetInfo(), module.WithServiceName(), @@ -45,7 +44,7 @@ var ( // been disabled to workaround the fact that Modules() will return // the static modules (not the dynamic ones) with a start delay. testModulesCreator = beater.Creator( - xpackbeater.WithLightModules(), + beater.WithLightModules(), beater.WithModuleOptions( module.WithMetricSetInfo(), module.WithMaxStartDelay(0), From c14cb8ba653cc3ce88fb6a65dc3591cc5854e1a0 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 1 Nov 2019 16:19:50 +0200 Subject: [PATCH 2/8] fixes Signed-off-by: chrismark --- .../metricbeat/mb/testdata/lightmodules/broken/module.yml | 3 --- .../metricbeat/mb/testdata/lightmodules/empty/.placeholder | 0 .../mb/testdata/lightmodules/service/metricset/manifest.yml | 6 ------ .../metricbeat/mb/testdata/lightmodules/service/module.yml | 4 ---- .../testdata/lightmodules/service/nondefault/manifest.yml | 4 ---- x-pack/metricbeat/scripts/msetlists/main.go | 3 +-- 6 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml delete mode 100644 x-pack/metricbeat/mb/testdata/lightmodules/empty/.placeholder delete mode 100644 x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml delete mode 100644 x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml delete mode 100644 x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml b/x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml deleted file mode 100644 index a4b4e73c3685..000000000000 --- a/x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml +++ /dev/null @@ -1,3 +0,0 @@ -name: broken -metricsets: -- notexists diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/empty/.placeholder b/x-pack/metricbeat/mb/testdata/lightmodules/empty/.placeholder deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml b/x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml deleted file mode 100644 index 5291cac44e68..000000000000 --- a/x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml +++ /dev/null @@ -1,6 +0,0 @@ -default: true -input: - module: foo - metricset: bar - defaults: - option: test diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml b/x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml deleted file mode 100644 index 7d037081eeb0..000000000000 --- a/x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml +++ /dev/null @@ -1,4 +0,0 @@ -name: service -metricsets: -- metricset -- nondefault diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml b/x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml deleted file mode 100644 index f4dc493d29c0..000000000000 --- a/x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml +++ /dev/null @@ -1,4 +0,0 @@ -default: false -input: - module: foo - metricset: baz diff --git a/x-pack/metricbeat/scripts/msetlists/main.go b/x-pack/metricbeat/scripts/msetlists/main.go index c5994e88b980..c1cb9ef1cc87 100644 --- a/x-pack/metricbeat/scripts/msetlists/main.go +++ b/x-pack/metricbeat/scripts/msetlists/main.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/beats/libbeat/paths" "github.com/elastic/beats/metricbeat/mb" _ "github.com/elastic/beats/x-pack/metricbeat/include" - xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" ) func main() { @@ -22,7 +21,7 @@ func main() { os.Setenv("BEAT_STRICT_PERMS", "false") path := paths.Resolve(paths.Home, "../x-pack/metricbeat/module") - lm := xpackmb.NewLightModulesSource(path) + lm := mb.NewLightModulesSource(path) mb.Registry.SetSecondarySource(lm) msList := msetlists.DefaultMetricsets() From 1410cf9649379f4fb1b6ac5a0e67168cf9794ff7 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 7 Nov 2019 14:32:45 -0500 Subject: [PATCH 3/8] fmt code Signed-off-by: chrismark --- metricbeat/beater/metricbeat.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 993401e46f4f..87036cfe2f4e 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -18,9 +18,10 @@ package beater import ( - "github.com/elastic/beats/libbeat/paths" "sync" + "github.com/elastic/beats/libbeat/paths" + "github.com/elastic/beats/libbeat/common/reload" "github.com/elastic/beats/libbeat/management" From 90ab9760f4308d4ac6d6790c491438c7c3d8cecf Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 7 Nov 2019 16:38:28 -0500 Subject: [PATCH 4/8] fix remaining references Signed-off-by: chrismark --- x-pack/metricbeat/module/aws/ebs/ebs_test.go | 3 +-- x-pack/metricbeat/module/aws/elb/elb_test.go | 3 +-- .../module/cockroachdb/status/status_integration_test.go | 4 +--- x-pack/metricbeat/module/cockroachdb/status/status_test.go | 3 +-- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/x-pack/metricbeat/module/aws/ebs/ebs_test.go b/x-pack/metricbeat/module/aws/ebs/ebs_test.go index 3017894f9455..6709a935d1d9 100644 --- a/x-pack/metricbeat/module/aws/ebs/ebs_test.go +++ b/x-pack/metricbeat/module/aws/ebs/ebs_test.go @@ -8,7 +8,6 @@ import ( "os" "github.com/elastic/beats/metricbeat/mb" - xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" // Register input module and metricset _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" @@ -18,5 +17,5 @@ import ( func init() { // To be moved to some kind of helper os.Setenv("BEAT_STRICT_PERMS", "false") - mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource("../../../module")) + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) } diff --git a/x-pack/metricbeat/module/aws/elb/elb_test.go b/x-pack/metricbeat/module/aws/elb/elb_test.go index eb91b04395d2..ed2632028acc 100644 --- a/x-pack/metricbeat/module/aws/elb/elb_test.go +++ b/x-pack/metricbeat/module/aws/elb/elb_test.go @@ -8,7 +8,6 @@ import ( "os" "github.com/elastic/beats/metricbeat/mb" - xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" // Register input module and metricset _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" @@ -18,5 +17,5 @@ import ( func init() { // To be moved to some kind of helper os.Setenv("BEAT_STRICT_PERMS", "false") - mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource("../../../module")) + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) } diff --git a/x-pack/metricbeat/module/cockroachdb/status/status_integration_test.go b/x-pack/metricbeat/module/cockroachdb/status/status_integration_test.go index ca1e1a9a22bf..2dd7a68ed611 100644 --- a/x-pack/metricbeat/module/cockroachdb/status/status_integration_test.go +++ b/x-pack/metricbeat/module/cockroachdb/status/status_integration_test.go @@ -15,8 +15,6 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" - // Register input module and metricset _ "github.com/elastic/beats/metricbeat/module/prometheus" _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" @@ -25,7 +23,7 @@ import ( func init() { // To be moved to some kind of helper os.Setenv("BEAT_STRICT_PERMS", "false") - mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource("../../../module")) + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) } func TestFetch(t *testing.T) { diff --git a/x-pack/metricbeat/module/cockroachdb/status/status_test.go b/x-pack/metricbeat/module/cockroachdb/status/status_test.go index b796fcf6d9c4..7002202b6087 100644 --- a/x-pack/metricbeat/module/cockroachdb/status/status_test.go +++ b/x-pack/metricbeat/module/cockroachdb/status/status_test.go @@ -13,7 +13,6 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" // Register input module and metricset _ "github.com/elastic/beats/metricbeat/module/prometheus" @@ -23,7 +22,7 @@ import ( func init() { // To be moved to some kind of helper os.Setenv("BEAT_STRICT_PERMS", "false") - mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource("../../../module")) + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) } func TestEventMapping(t *testing.T) { From 9a1dda1bc546107c963637e23ef59301734c9263 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 8 Nov 2019 16:55:13 -0500 Subject: [PATCH 5/8] Add changelog entry Signed-off-by: chrismark --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 3510a97e8420..10f92aa8f835 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -26,6 +26,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509] - The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610] - Altered all remaining uses of mapval to use the renamed and enhanced version: https://github.com/elastic/go-lookslike[go-lookslike] instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165] +- Move light modules to OSS. {pull}14369[14369] ==== Bugfixes From 1ed075fc9584ec979bac9adfa92ff8358ed1dfc6 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 12 Nov 2019 00:02:56 +0200 Subject: [PATCH 6/8] Update metricbeat/beater/metricbeat.go Co-Authored-By: Jaime Soriano Pastor --- metricbeat/beater/metricbeat.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 87036cfe2f4e..ac2b09169096 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -21,7 +21,6 @@ import ( "sync" "github.com/elastic/beats/libbeat/paths" - "github.com/elastic/beats/libbeat/common/reload" "github.com/elastic/beats/libbeat/management" From 51725366ae4f4caafa38bfbc51c2df3f2d814a1c Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 12 Nov 2019 00:35:24 +0200 Subject: [PATCH 7/8] Make use of DefaultCreator in x-pack metricbeat inti Signed-off-by: chrismark --- metricbeat/beater/metricbeat.go | 2 +- x-pack/metricbeat/cmd/root.go | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index ac2b09169096..2308b5fe6620 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -20,9 +20,9 @@ package beater import ( "sync" - "github.com/elastic/beats/libbeat/paths" "github.com/elastic/beats/libbeat/common/reload" "github.com/elastic/beats/libbeat/management" + "github.com/elastic/beats/libbeat/paths" "github.com/joeshaw/multierror" "github.com/pkg/errors" diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 5c3451e03209..403892bb69dd 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -32,14 +32,6 @@ var Name = "metricbeat" var RootCmd *cmd.BeatsRootCmd var ( - rootCreator = beater.Creator( - beater.WithLightModules(), - beater.WithModuleOptions( - module.WithMetricSetInfo(), - module.WithServiceName(), - ), - ) - // Use a customized instance of Metricbeat where startup delay has // been disabled to workaround the fact that Modules() will return // the static modules (not the dynamic ones) with a start delay. @@ -55,7 +47,7 @@ var ( func init() { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) - RootCmd = cmd.GenRootCmdWithSettings(rootCreator, instance.Settings{RunFlags: runFlags, Name: Name}) + RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{RunFlags: runFlags, Name: Name}) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator)) xpackcmd.AddXPack(RootCmd, Name) From 21f843ecc962ecead22adfae615bd3584ee8b41a Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 12 Nov 2019 00:43:52 +0200 Subject: [PATCH 8/8] Add DefaultTestModulesCreator method Signed-off-by: chrismark --- metricbeat/beater/metricbeat.go | 24 ++++++++++++++++++++++++ metricbeat/cmd/root.go | 19 ++----------------- x-pack/metricbeat/cmd/root.go | 18 ++---------------- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 2308b5fe6620..f071d6e7bffb 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -107,6 +107,30 @@ func DefaultCreator() beat.Creator { ) } +// DefaultTestModulesCreator returns a customized instance of Metricbeat +// where startup delay has been disabled to workaround the fact that +// Modules() will return the static modules (not the dynamic ones) +// with a start delay. +// +// This is equivalent to calling +// +// beater.Creator( +// beater.WithLightModules(), +// beater.WithModuleOptions( +// module.WithMetricSetInfo(), +// module.WithMaxStartDelay(0), +// ), +// ) +func DefaultTestModulesCreator() beat.Creator { + return Creator( + WithLightModules(), + WithModuleOptions( + module.WithMetricSetInfo(), + module.WithMaxStartDelay(0), + ), + ) +} + // newMetricbeat creates and returns a new Metricbeat instance. func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbeat, error) { config := defaultConfig diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index 4fe2bf966f6c..d54b72cb4648 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -22,12 +22,10 @@ import ( "github.com/spf13/pflag" - cmd "github.com/elastic/beats/libbeat/cmd" + "github.com/elastic/beats/libbeat/cmd" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/metricbeat/beater" "github.com/elastic/beats/metricbeat/cmd/test" - "github.com/elastic/beats/metricbeat/mb/module" - // import modules _ "github.com/elastic/beats/metricbeat/include" _ "github.com/elastic/beats/metricbeat/include/fields" @@ -39,23 +37,10 @@ var Name = "metricbeat" // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd -var ( - // Use a customized instance of Metricbeat where startup delay has - // been disabled to workaround the fact that Modules() will return - // the static modules (not the dynamic ones) with a start delay. - testModulesCreator = beater.Creator( - beater.WithLightModules(), - beater.WithModuleOptions( - module.WithMetricSetInfo(), - module.WithMaxStartDelay(0), - ), - ) -) - func init() { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{RunFlags: runFlags, Name: Name}) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager)) - RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator)) + RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) } diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 403892bb69dd..0acd6dbdc01c 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -9,12 +9,11 @@ import ( "github.com/spf13/pflag" - cmd "github.com/elastic/beats/libbeat/cmd" + "github.com/elastic/beats/libbeat/cmd" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/metricbeat/beater" mbcmd "github.com/elastic/beats/metricbeat/cmd" "github.com/elastic/beats/metricbeat/cmd/test" - "github.com/elastic/beats/metricbeat/mb/module" xpackcmd "github.com/elastic/beats/x-pack/libbeat/cmd" // Register the includes. @@ -31,24 +30,11 @@ var Name = "metricbeat" // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd -var ( - // Use a customized instance of Metricbeat where startup delay has - // been disabled to workaround the fact that Modules() will return - // the static modules (not the dynamic ones) with a start delay. - testModulesCreator = beater.Creator( - beater.WithLightModules(), - beater.WithModuleOptions( - module.WithMetricSetInfo(), - module.WithMaxStartDelay(0), - ), - ) -) - func init() { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{RunFlags: runFlags, Name: Name}) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) - RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator)) + RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) xpackcmd.AddXPack(RootCmd, Name) }