From 1662859c98c48baedb3d4375d744c7ece69867bd Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Tue, 16 Jan 2018 18:50:32 +0500 Subject: [PATCH 01/10] add diskio and raid filtering using regular expressions --- metricbeat/docs/modules/system.asciidoc | 2 ++ metricbeat/metricbeat.reference.yml | 6 +++++ .../module/system/_meta/config.reference.yml | 6 +++++ metricbeat/module/system/_meta/config.yml | 2 ++ .../module/system/diskio/_meta/docs.asciidoc | 14 ++++++++++- .../diskio/_meta/testdata/proc/diskstats | 10 ++++++++ metricbeat/module/system/diskio/diskio.go | 25 +++++++++++++++++++ .../module/system/diskio/diskio_test.go | 24 ++++++++++++++++++ .../module/system/raid/_meta/docs.asciidoc | 17 +++++++++++++ .../system/raid/_meta/testdata/proc/mdstat | 4 +++ metricbeat/module/system/raid/raid.go | 23 +++++++++++++++-- metricbeat/module/system/raid/raid_test.go | 11 +++++++- metricbeat/modules.d/system.yml | 2 ++ 13 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 metricbeat/module/system/diskio/_meta/testdata/proc/diskstats diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 56f36d6cde0..8c70f39fbd2 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -41,6 +41,8 @@ metricbeat.modules: process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + #diskio.name.regexp: ^sda + #raid.name.regexp: ^md - module: system period: 1m diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index a864f20f6e7..65c33a4c56e 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -111,6 +111,12 @@ metricbeat.modules: #socket.reverse_lookup.success_ttl: 60s #socket.reverse_lookup.failure_ttl: 60s + # Diskio configurations + #diskio.name.regexp: ^sda + + # RAID configurations + #raid.name.regexp: ^md + #------------------------------ Aerospike Module ----------------------------- - module: aerospike metricsets: ["namespace"] diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 17dedccbe4c..368f7a1741c 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -60,3 +60,9 @@ #socket.reverse_lookup.enabled: false #socket.reverse_lookup.success_ttl: 60s #socket.reverse_lookup.failure_ttl: 60s + + # Diskio configurations + #diskio.name.regexp: ^sda + + # RAID configurations + #raid.name.regexp: ^md diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index 409ec1b636c..d2c114a743b 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -14,6 +14,8 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + #diskio.name.regexp: ^sda + #raid.name.regexp: ^md - module: system period: 1m diff --git a/metricbeat/module/system/diskio/_meta/docs.asciidoc b/metricbeat/module/system/diskio/_meta/docs.asciidoc index fe8da1ce3c7..0716ed786f3 100644 --- a/metricbeat/module/system/diskio/_meta/docs.asciidoc +++ b/metricbeat/module/system/diskio/_meta/docs.asciidoc @@ -11,4 +11,16 @@ This metricset is available on: [float] === Configuration -There are no configuration options for this metricset. +*`diskio.name.regexp`*:: When the `diskio` metricset is enabled, you can use the +`diskio.name.regexp` option to define a list of regexp expressions to filter the +disk devices that are reported. If not set all disk devices are returned ++ +The following example config returns metrics for devices starting with name sda: ++ +[source,yaml] +---- +metricbeat.modules: +- module: system + metricsets: ["diskio"] + diskio.name.regexp: ^sda +---- diff --git a/metricbeat/module/system/diskio/_meta/testdata/proc/diskstats b/metricbeat/module/system/diskio/_meta/testdata/proc/diskstats new file mode 100644 index 00000000000..8423e5c22e1 --- /dev/null +++ b/metricbeat/module/system/diskio/_meta/testdata/proc/diskstats @@ -0,0 +1,10 @@ + 8 0 sda 592 0 4730 4924 0 0 0 0 0 1496 4924 + 8 1 sda1 85 0 680 204 0 0 0 0 0 204 204 + 8 2 sda2 81 0 648 1056 0 0 0 0 0 1056 1056 + 8 3 sda3 81 0 648 1044 0 0 0 0 0 1044 1044 + 8 4 sda4 1 0 2 0 0 0 0 0 0 0 0 + 8 5 sda5 80 0 640 1164 0 0 0 0 0 1164 1164 + 8 6 sda6 82 0 656 1296 0 0 0 0 0 1296 1296 + 8 16 sdb 7681508 3044909 164279706 2902996 9494419 12814534 568199104 61998668 0 3842396 64959416 + 8 17 sdb1 3146154 2988374 49078184 833872 861040 7368290 67190704 11224168 0 601740 12075776 + 8 18 sdb2 4535191 56535 115200218 2069120 7703033 5446244 501008400 48646568 0 2440248 50884632 diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index a0a8444dc8c..f880fa5f6fb 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -4,9 +4,12 @@ package diskio import ( "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "regexp" + "github.com/pkg/errors" "github.com/shirou/gopsutil/disk" ) @@ -21,14 +24,33 @@ func init() { type MetricSet struct { mb.BaseMetricSet statistics *DiskIOStat + nameRegexp *regexp.Regexp } // New is a mb.MetricSetFactory that returns a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := struct { + Regexp string `config:"diskio.name.regexp"` + }{Regexp: ""} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + ms := &MetricSet{ BaseMetricSet: base, statistics: NewDiskIOStat(), } + + if len(config.Regexp) > 0 { + r, err := regexp.Compile(config.Regexp) + if err != nil { + logp.Warn("diskio", "Invalid regular expression: (%s), error: %s", config.Regexp, err.Error()) + return nil, err + } + ms.nameRegexp = r + } + return ms, nil } @@ -44,6 +66,9 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { events := make([]common.MapStr, 0, len(stats)) for _, counters := range stats { + if m.nameRegexp != nil && !m.nameRegexp.MatchString(counters.Name) { + continue + } event := common.MapStr{ "name": counters.Name, diff --git a/metricbeat/module/system/diskio/diskio_test.go b/metricbeat/module/system/diskio/diskio_test.go index 4d5d8817d27..3b7f9eff54c 100644 --- a/metricbeat/module/system/diskio/diskio_test.go +++ b/metricbeat/module/system/diskio/diskio_test.go @@ -6,7 +6,10 @@ package diskio import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/system" ) func TestData(t *testing.T) { @@ -17,6 +20,27 @@ func TestData(t *testing.T) { } } +func TestDataRegexp(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := getConfig() + conf["diskio.name.regexp"] = "^sda" + f := mbtest.NewEventsFetcher(t, getConfig()) + + if err := mbtest.WriteEvents(f, t); err != nil { + t.Fatal("write", err) + } + + data, err := f.Fetch() + assert.NoError(t, err) + assert.Equal(t, 7, len(data)) +} + func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "system", diff --git a/metricbeat/module/system/raid/_meta/docs.asciidoc b/metricbeat/module/system/raid/_meta/docs.asciidoc index 6544b9da599..048e299a154 100644 --- a/metricbeat/module/system/raid/_meta/docs.asciidoc +++ b/metricbeat/module/system/raid/_meta/docs.asciidoc @@ -1,3 +1,20 @@ === system raid MetricSet This is the raid metricset of the module system. + +[float] +=== Configuration + +*`raid.name.regexp`*:: When the `raid` metricset is enabled, you can use the +`raid.name.regexp` option to define a list of regexp expressions to filter the +RAID controllers that are reported. If not set all controllers are returned ++ +The following example config returns metrics for controllers starting with name md1: ++ +[source,yaml] +---- +metricbeat.modules: +- module: system + metricsets: ["raid"] + raid.name.regexp: ^md1 +---- diff --git a/metricbeat/module/system/raid/_meta/testdata/proc/mdstat b/metricbeat/module/system/raid/_meta/testdata/proc/mdstat index 162aa1b984f..459c37ee2b4 100644 --- a/metricbeat/module/system/raid/_meta/testdata/proc/mdstat +++ b/metricbeat/module/system/raid/_meta/testdata/proc/mdstat @@ -23,4 +23,8 @@ md7 : active raid6 sdb1[0] sde1[3] sdd1[2] sdc1[1] 7813735424 blocks super 1.2 level 6, 512k chunk, algorithm 2 [4/3] [U_UU] bitmap: 0/30 pages [0KB], 65536KB chunk +md1 : active raid6 sdb1[0] sde1[3] sdd1[2] sdc1[1] + 7813735424 blocks super 1.2 level 6, 512k chunk, algorithm 2 [4/3] [U_UU] + bitmap: 0/30 pages [0KB], 65536KB chunk + unused devices: diff --git a/metricbeat/module/system/raid/raid.go b/metricbeat/module/system/raid/raid.go index 23eae508ed7..423d309ce2d 100644 --- a/metricbeat/module/system/raid/raid.go +++ b/metricbeat/module/system/raid/raid.go @@ -2,9 +2,11 @@ package raid import ( "path/filepath" + "regexp" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/system" @@ -22,7 +24,8 @@ func init() { // MetricSet contains proc fs data. type MetricSet struct { mb.BaseMetricSet - fs procfs.FS + fs procfs.FS + nameRegexp *regexp.Regexp } // New creates a new instance of the raid metricset. @@ -37,7 +40,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Additional configuration options config := struct { MountPoint string `config:"raid.mount_point"` - }{} + Regexp string `config:"raid.name.regexp"` + }{ + Regexp: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -58,6 +64,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { fs: fs, } + if len(config.Regexp) > 0 { + r, err := regexp.Compile(config.Regexp) + if err != nil { + logp.Warn("raid", "Invalid regular expression: (%s), error: %s", config.Regexp, err.Error()) + return nil, err + } + m.nameRegexp = r + } + return m, nil } @@ -71,6 +86,10 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { events := make([]common.MapStr, 0, len(stats)) for _, stat := range stats { + if m.nameRegexp != nil && !m.nameRegexp.MatchString(stat.Name) { + continue + } + event := common.MapStr{ "name": stat.Name, "activity_state": stat.ActivityState, diff --git a/metricbeat/module/system/raid/raid_test.go b/metricbeat/module/system/raid/raid_test.go index bb5fb15027b..729e54ac7a0 100644 --- a/metricbeat/module/system/raid/raid_test.go +++ b/metricbeat/module/system/raid/raid_test.go @@ -20,7 +20,16 @@ func TestFetch(t *testing.T) { f := mbtest.NewEventsFetcher(t, getConfig()) data, err := f.Fetch() assert.NoError(t, err) - assert.Equal(t, 7, len(data)) + assert.Equal(t, 8, len(data)) +} + +func TestFetchRegexp(t *testing.T) { + conf := getConfig() + conf["raid.name.regexp"] = "^md1" + f := mbtest.NewEventsFetcher(t, conf) + data, err := f.Fetch() + assert.NoError(t, err) + assert.Equal(t, 2, len(data)) } func getConfig() map[string]interface{} { diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 409ec1b636c..d2c114a743b 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -14,6 +14,8 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + #diskio.name.regexp: ^sda + #raid.name.regexp: ^md - module: system period: 1m From 56f4f71f60d3670cdb467fb2bd1a4c25f68cb66e Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Sat, 20 Jan 2018 15:08:46 +0500 Subject: [PATCH 02/10] use only exact matched device names in diskio --- metricbeat/docs/modules/system.asciidoc | 3 +- metricbeat/metricbeat.reference.yml | 5 +--- .../module/system/_meta/config.reference.yml | 5 +--- metricbeat/module/system/_meta/config.yml | 3 +- .../module/system/diskio/_meta/docs.asciidoc | 11 +++---- metricbeat/module/system/diskio/diskio.go | 28 ++++-------------- .../module/system/diskio/diskio_test.go | 10 +++++-- .../module/system/raid/_meta/docs.asciidoc | 17 ----------- metricbeat/module/system/raid/raid.go | 29 +++---------------- metricbeat/module/system/raid/raid_test.go | 9 ------ metricbeat/modules.d/system.yml | 3 +- 11 files changed, 28 insertions(+), 95 deletions(-) diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 8c70f39fbd2..f91e2631bd0 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -41,8 +41,7 @@ metricbeat.modules: process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.name.regexp: ^sda - #raid.name.regexp: ^md + #diskio.include_devices: ["sda", "sda1", "sda2"] - module: system period: 1m diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 65c33a4c56e..af69f62abba 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -112,10 +112,7 @@ metricbeat.modules: #socket.reverse_lookup.failure_ttl: 60s # Diskio configurations - #diskio.name.regexp: ^sda - - # RAID configurations - #raid.name.regexp: ^md + #diskio.include_devices: ["sda", "sda1", "sda2"] #------------------------------ Aerospike Module ----------------------------- - module: aerospike diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 368f7a1741c..cc7e6e039ef 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -62,7 +62,4 @@ #socket.reverse_lookup.failure_ttl: 60s # Diskio configurations - #diskio.name.regexp: ^sda - - # RAID configurations - #raid.name.regexp: ^md + #diskio.include_devices: ["sda", "sda1", "sda2"] diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index d2c114a743b..0326123d22d 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -14,8 +14,7 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.name.regexp: ^sda - #raid.name.regexp: ^md + #diskio.include_devices: ["sda", "sda1", "sda2"] - module: system period: 1m diff --git a/metricbeat/module/system/diskio/_meta/docs.asciidoc b/metricbeat/module/system/diskio/_meta/docs.asciidoc index 0716ed786f3..5f8fb092b9f 100644 --- a/metricbeat/module/system/diskio/_meta/docs.asciidoc +++ b/metricbeat/module/system/diskio/_meta/docs.asciidoc @@ -11,16 +11,17 @@ This metricset is available on: [float] === Configuration -*`diskio.name.regexp`*:: When the `diskio` metricset is enabled, you can use the -`diskio.name.regexp` option to define a list of regexp expressions to filter the -disk devices that are reported. If not set all disk devices are returned +*`diskio.include_devices`*:: When the `diskio` metricset is enabled, you can use the +`diskio.include_devices` option to define a list of device names to pre-filter the +devices that are reported. Filters only exact matches +If not set, all disk devices are returned + -The following example config returns metrics for devices starting with name sda: +The following example config returns metrics for devices matching include_devices: + [source,yaml] ---- metricbeat.modules: - module: system metricsets: ["diskio"] - diskio.name.regexp: ^sda + diskio.include_devices: ["sda", "sda1"] ---- diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index f880fa5f6fb..e8ae34f16ef 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -4,12 +4,9 @@ package diskio import ( "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" - "regexp" - "github.com/pkg/errors" "github.com/shirou/gopsutil/disk" ) @@ -24,34 +21,24 @@ func init() { type MetricSet struct { mb.BaseMetricSet statistics *DiskIOStat - nameRegexp *regexp.Regexp + names []string } // New is a mb.MetricSetFactory that returns a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { - Regexp string `config:"diskio.name.regexp"` - }{Regexp: ""} + Names []string `config:"diskio.include_devices"` + }{Names: []string{}} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - ms := &MetricSet{ + return &MetricSet{ BaseMetricSet: base, statistics: NewDiskIOStat(), - } - - if len(config.Regexp) > 0 { - r, err := regexp.Compile(config.Regexp) - if err != nil { - logp.Warn("diskio", "Invalid regular expression: (%s), error: %s", config.Regexp, err.Error()) - return nil, err - } - ms.nameRegexp = r - } - - return ms, nil + names: config.Names, + }, nil } // Fetch fetches disk IO metrics from the OS. @@ -66,9 +53,6 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { events := make([]common.MapStr, 0, len(stats)) for _, counters := range stats { - if m.nameRegexp != nil && !m.nameRegexp.MatchString(counters.Name) { - continue - } event := common.MapStr{ "name": counters.Name, diff --git a/metricbeat/module/system/diskio/diskio_test.go b/metricbeat/module/system/diskio/diskio_test.go index 3b7f9eff54c..55c064cf12d 100644 --- a/metricbeat/module/system/diskio/diskio_test.go +++ b/metricbeat/module/system/diskio/diskio_test.go @@ -18,9 +18,13 @@ func TestData(t *testing.T) { if err := mbtest.WriteEvents(f, t); err != nil { t.Fatal("write", err) } + + data, err := f.Fetch() + assert.NoError(t, err) + assert.Equal(t, 10, len(data)) } -func TestDataRegexp(t *testing.T) { +func TestDataNameFilter(t *testing.T) { oldFS := system.HostFS newFS := "_meta/testdata" system.HostFS = &newFS @@ -29,7 +33,7 @@ func TestDataRegexp(t *testing.T) { }() conf := getConfig() - conf["diskio.name.regexp"] = "^sda" + conf["diskio.include_devices"] = []string{"sda", "sda1", "sda2"} f := mbtest.NewEventsFetcher(t, getConfig()) if err := mbtest.WriteEvents(f, t); err != nil { @@ -38,7 +42,7 @@ func TestDataRegexp(t *testing.T) { data, err := f.Fetch() assert.NoError(t, err) - assert.Equal(t, 7, len(data)) + assert.Equal(t, 3, len(data)) } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/raid/_meta/docs.asciidoc b/metricbeat/module/system/raid/_meta/docs.asciidoc index 048e299a154..6544b9da599 100644 --- a/metricbeat/module/system/raid/_meta/docs.asciidoc +++ b/metricbeat/module/system/raid/_meta/docs.asciidoc @@ -1,20 +1,3 @@ === system raid MetricSet This is the raid metricset of the module system. - -[float] -=== Configuration - -*`raid.name.regexp`*:: When the `raid` metricset is enabled, you can use the -`raid.name.regexp` option to define a list of regexp expressions to filter the -RAID controllers that are reported. If not set all controllers are returned -+ -The following example config returns metrics for controllers starting with name md1: -+ -[source,yaml] ----- -metricbeat.modules: -- module: system - metricsets: ["raid"] - raid.name.regexp: ^md1 ----- diff --git a/metricbeat/module/system/raid/raid.go b/metricbeat/module/system/raid/raid.go index 423d309ce2d..1fc032431fe 100644 --- a/metricbeat/module/system/raid/raid.go +++ b/metricbeat/module/system/raid/raid.go @@ -2,11 +2,9 @@ package raid import ( "path/filepath" - "regexp" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/system" @@ -24,8 +22,7 @@ func init() { // MetricSet contains proc fs data. type MetricSet struct { mb.BaseMetricSet - fs procfs.FS - nameRegexp *regexp.Regexp + fs procfs.FS } // New creates a new instance of the raid metricset. @@ -40,10 +37,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Additional configuration options config := struct { MountPoint string `config:"raid.mount_point"` - Regexp string `config:"raid.name.regexp"` - }{ - Regexp: "", - } + }{} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -59,21 +53,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } - m := &MetricSet{ + return &MetricSet{ BaseMetricSet: base, fs: fs, - } - - if len(config.Regexp) > 0 { - r, err := regexp.Compile(config.Regexp) - if err != nil { - logp.Warn("raid", "Invalid regular expression: (%s), error: %s", config.Regexp, err.Error()) - return nil, err - } - m.nameRegexp = r - } - - return m, nil + }, nil } // Fetch fetches one event for each device @@ -86,10 +69,6 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { events := make([]common.MapStr, 0, len(stats)) for _, stat := range stats { - if m.nameRegexp != nil && !m.nameRegexp.MatchString(stat.Name) { - continue - } - event := common.MapStr{ "name": stat.Name, "activity_state": stat.ActivityState, diff --git a/metricbeat/module/system/raid/raid_test.go b/metricbeat/module/system/raid/raid_test.go index 729e54ac7a0..9c2e2a58d5e 100644 --- a/metricbeat/module/system/raid/raid_test.go +++ b/metricbeat/module/system/raid/raid_test.go @@ -23,15 +23,6 @@ func TestFetch(t *testing.T) { assert.Equal(t, 8, len(data)) } -func TestFetchRegexp(t *testing.T) { - conf := getConfig() - conf["raid.name.regexp"] = "^md1" - f := mbtest.NewEventsFetcher(t, conf) - data, err := f.Fetch() - assert.NoError(t, err) - assert.Equal(t, 2, len(data)) -} - func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "system", diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index d2c114a743b..0326123d22d 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -14,8 +14,7 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.name.regexp: ^sda - #raid.name.regexp: ^md + #diskio.include_devices: ["sda", "sda1", "sda2"] - module: system period: 1m From 081aebd9f141293d0f26778dfe06b4ddcb90dd65 Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Sat, 20 Jan 2018 15:35:51 +0500 Subject: [PATCH 03/10] filter IOCounters, move tests into linux arch --- metricbeat/module/system/diskio/diskio.go | 2 +- .../module/system/diskio/diskio_test.go | 28 ---------- .../system/diskio/diskstat_linux_test.go | 51 +++++++++++++++++++ 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index e8ae34f16ef..8e36292b1fd 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -43,7 +43,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch fetches disk IO metrics from the OS. func (m *MetricSet) Fetch() ([]common.MapStr, error) { - stats, err := disk.IOCounters() + stats, err := disk.IOCounters(m.names...) if err != nil { return nil, errors.Wrap(err, "disk io counters") } diff --git a/metricbeat/module/system/diskio/diskio_test.go b/metricbeat/module/system/diskio/diskio_test.go index 55c064cf12d..4d5d8817d27 100644 --- a/metricbeat/module/system/diskio/diskio_test.go +++ b/metricbeat/module/system/diskio/diskio_test.go @@ -6,10 +6,7 @@ package diskio import ( "testing" - "github.com/stretchr/testify/assert" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/system" ) func TestData(t *testing.T) { @@ -18,31 +15,6 @@ func TestData(t *testing.T) { if err := mbtest.WriteEvents(f, t); err != nil { t.Fatal("write", err) } - - data, err := f.Fetch() - assert.NoError(t, err) - assert.Equal(t, 10, len(data)) -} - -func TestDataNameFilter(t *testing.T) { - oldFS := system.HostFS - newFS := "_meta/testdata" - system.HostFS = &newFS - defer func() { - system.HostFS = oldFS - }() - - conf := getConfig() - conf["diskio.include_devices"] = []string{"sda", "sda1", "sda2"} - f := mbtest.NewEventsFetcher(t, getConfig()) - - if err := mbtest.WriteEvents(f, t); err != nil { - t.Fatal("write", err) - } - - data, err := f.Fetch() - assert.NoError(t, err) - assert.Equal(t, 3, len(data)) } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/diskio/diskstat_linux_test.go b/metricbeat/module/system/diskio/diskstat_linux_test.go index ff806e5cba1..3b12475104b 100644 --- a/metricbeat/module/system/diskio/diskstat_linux_test.go +++ b/metricbeat/module/system/diskio/diskstat_linux_test.go @@ -5,6 +5,8 @@ package diskio import ( "testing" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/system" "github.com/stretchr/testify/assert" ) @@ -12,3 +14,52 @@ func Test_Get_CLK_TCK(t *testing.T) { //usually the tick is 100 assert.Equal(t, uint32(100), Get_CLK_TCK()) } + +func TestDataNameFilter(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + "diskio.include_devices": []string{"sda", "sda1", "sda2"}, + } + + f := mbtest.NewEventsFetcher(t, conf) + + if err := mbtest.WriteEvents(f, t); err != nil { + t.Fatal("write", err) + } + + data, err := f.Fetch() + assert.NoError(t, err) + assert.Equal(t, 3, len(data)) +} + +func TestDataEmptyFilter(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + } + + f := mbtest.NewEventsFetcher(t, conf) + + if err := mbtest.WriteEvents(f, t); err != nil { + t.Fatal("write", err) + } + + data, err := f.Fetch() + assert.NoError(t, err) + assert.Equal(t, 10, len(data)) +} From f44b21bfe0876b8653d52fed91ea569409cd137a Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Mon, 22 Jan 2018 09:49:18 +0500 Subject: [PATCH 04/10] add entry to CHANGELOG --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 653b82e8aeb..46842553272 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Support haproxy stats gathering using http (additionaly to tcp socket). {pull}5819[5819] - De dot keys in jolokia/jmx metricset to prevent collisions. {pull}5957[5957] - Support to optionally 'de dot' keys in http/json metricset to prevent collisions. {pull}5957[5957] +- Add filtering option by exact device names in system.diskio. `diskio.include_devices`. {pull}6085[6085] *Packetbeat* From 0ab80829518c8f7e4ad3dc22e44439d2ea0c1b9b Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Mon, 22 Jan 2018 10:00:26 +0500 Subject: [PATCH 05/10] coding style fix --- metricbeat/module/system/diskio/diskstat_linux_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/system/diskio/diskstat_linux_test.go b/metricbeat/module/system/diskio/diskstat_linux_test.go index 3b12475104b..342597dcfad 100644 --- a/metricbeat/module/system/diskio/diskstat_linux_test.go +++ b/metricbeat/module/system/diskio/diskstat_linux_test.go @@ -5,9 +5,10 @@ package diskio import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/system" - "github.com/stretchr/testify/assert" ) func Test_Get_CLK_TCK(t *testing.T) { From a9c22bcd3f8fde204083ec761cffcfa1a7c60557 Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Fri, 26 Jan 2018 09:20:23 +0500 Subject: [PATCH 06/10] make default config fields empty --- metricbeat/docs/modules/system.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 2 +- .../module/system/_meta/config.reference.yml | 2 +- metricbeat/module/system/_meta/config.yml | 2 +- .../module/system/diskio/_meta/docs.asciidoc | 4 ++-- metricbeat/module/system/diskio/diskio.go | 16 ++++++++-------- metricbeat/modules.d/system.yml | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index f91e2631bd0..b1aa50dcb97 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -41,7 +41,7 @@ metricbeat.modules: process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.include_devices: ["sda", "sda1", "sda2"] + #diskio.include_devices: [] - module: system period: 1m diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 6773907d1cd..66110c5b540 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -112,7 +112,7 @@ metricbeat.modules: #socket.reverse_lookup.failure_ttl: 60s # Diskio configurations - #diskio.include_devices: ["sda", "sda1", "sda2"] + #diskio.include_devices: [] #------------------------------ Aerospike Module ----------------------------- - module: aerospike diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index cc7e6e039ef..b8d5f9106aa 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -62,4 +62,4 @@ #socket.reverse_lookup.failure_ttl: 60s # Diskio configurations - #diskio.include_devices: ["sda", "sda1", "sda2"] + #diskio.include_devices: [] diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index 0326123d22d..ad41e5c94b8 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -14,7 +14,7 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.include_devices: ["sda", "sda1", "sda2"] + #diskio.include_devices: [] - module: system period: 1m diff --git a/metricbeat/module/system/diskio/_meta/docs.asciidoc b/metricbeat/module/system/diskio/_meta/docs.asciidoc index 5f8fb092b9f..cce8b162b5f 100644 --- a/metricbeat/module/system/diskio/_meta/docs.asciidoc +++ b/metricbeat/module/system/diskio/_meta/docs.asciidoc @@ -13,8 +13,8 @@ This metricset is available on: *`diskio.include_devices`*:: When the `diskio` metricset is enabled, you can use the `diskio.include_devices` option to define a list of device names to pre-filter the -devices that are reported. Filters only exact matches -If not set, all disk devices are returned +devices that are reported. Filters only exact matches. +If not set or given `[]` empty array, all disk devices are returned + The following example config returns metrics for devices matching include_devices: + diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 8e36292b1fd..085b7ed703d 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -20,30 +20,30 @@ func init() { // MetricSet for fetching system disk IO metrics. type MetricSet struct { mb.BaseMetricSet - statistics *DiskIOStat - names []string + statistics *DiskIOStat + includeDevices []string } // New is a mb.MetricSetFactory that returns a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { - Names []string `config:"diskio.include_devices"` - }{Names: []string{}} + IncludeDevices []string `config:"diskio.include_devices"` + }{IncludeDevices: []string{}} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } return &MetricSet{ - BaseMetricSet: base, - statistics: NewDiskIOStat(), - names: config.Names, + BaseMetricSet: base, + statistics: NewDiskIOStat(), + includeDevices: config.IncludeDevices, }, nil } // Fetch fetches disk IO metrics from the OS. func (m *MetricSet) Fetch() ([]common.MapStr, error) { - stats, err := disk.IOCounters(m.names...) + stats, err := disk.IOCounters(m.includeDevices...) if err != nil { return nil, errors.Wrap(err, "disk io counters") } diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 0326123d22d..ad41e5c94b8 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -14,7 +14,7 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.include_devices: ["sda", "sda1", "sda2"] + #diskio.include_devices: [] - module: system period: 1m From fde8640caccf7a981680105f47342d12b6750e1c Mon Sep 17 00:00:00 2001 From: Ikhtiyor <33823221+iahmedov@users.noreply.github.com> Date: Sat, 3 Feb 2018 18:44:26 +0500 Subject: [PATCH 07/10] remove default config --- metricbeat/modules.d/system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index ad41e5c94b8..409ec1b636c 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -14,7 +14,6 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.include_devices: [] - module: system period: 1m From d3999b6fafd6e61a69d819b5d80a570827957808 Mon Sep 17 00:00:00 2001 From: Ikhtiyor <33823221+iahmedov@users.noreply.github.com> Date: Sat, 3 Feb 2018 18:44:35 +0500 Subject: [PATCH 08/10] remove default config --- metricbeat/module/system/_meta/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index ad41e5c94b8..409ec1b636c 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -14,7 +14,6 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.include_devices: [] - module: system period: 1m From df020ff488e02a02bee9b5dbcaba248f4ac41387 Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Sun, 4 Feb 2018 14:39:20 +0500 Subject: [PATCH 09/10] fix changelog --- CHANGELOG.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 71d78b8c33c..77f74c0b4ee 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -51,10 +51,9 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Rename `golang.heap.system.optained` field to `golang.heap.system.obtained`. {issue}5703[5703] - Support haproxy stats gathering using http (additionaly to tcp socket). {pull}5819[5819] - De dot keys in jolokia/jmx metricset to prevent collisions. {pull}5957[5957] -- Support to optionally 'de dot' keys in http/json metricset to prevent collisions. {pull}5957[5957] -- Add filtering option by exact device names in system.diskio. `diskio.include_devices`. {pull}6085[6085] - Support to optionally 'de dot' keys in http/json metricset to prevent collisions. {pull}5970[5970] - De dot keys in kubernetes/event metricset to prevent collisions. {pull}6203[6203] +- Add filtering option by exact device names in system.diskio. `diskio.include_devices`. {pull}6085[6085] *Packetbeat* From 2d7b02f2cf8d64aa8a551f3c33a2a32172a58b2d Mon Sep 17 00:00:00 2001 From: Ikhtiyor Ahmedov Date: Tue, 6 Feb 2018 11:10:02 +0500 Subject: [PATCH 10/10] update autogenerated content --- metricbeat/docs/modules/system.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index b1aa50dcb97..56f36d6cde0 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -41,7 +41,6 @@ metricbeat.modules: process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - #diskio.include_devices: [] - module: system period: 1m