From 758b7df5631066b118f8ab4a1f5b7796f7de76c9 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 9 Jan 2024 12:20:07 -0800 Subject: [PATCH] [receiver/filestats] add file.count metric (#30191) **Description:** Add a file.count metric to filestatsreceiver that reports the number of files matched by the receiver **Link to tracking Issue:** Fixes #24651 --------- Co-authored-by: Dmitrii Anoshin --- .chloggen/filestats_filecount.yaml | 27 +++++++++ receiver/filestatsreceiver/documentation.md | 8 +++ .../internal/metadata/generated_config.go | 4 ++ .../metadata/generated_config_test.go | 2 + .../internal/metadata/generated_metrics.go | 57 +++++++++++++++++++ .../metadata/generated_metrics_test.go | 15 +++++ .../internal/metadata/testdata/config.yaml | 4 ++ receiver/filestatsreceiver/metadata.yaml | 6 ++ receiver/filestatsreceiver/scraper.go | 3 + receiver/filestatsreceiver/scraper_test.go | 12 +++- 10 files changed, 136 insertions(+), 2 deletions(-) create mode 100755 .chloggen/filestats_filecount.yaml diff --git a/.chloggen/filestats_filecount.yaml b/.chloggen/filestats_filecount.yaml new file mode 100755 index 000000000000..589564adece8 --- /dev/null +++ b/.chloggen/filestats_filecount.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: filestatsreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add a file.count metric to filestatsreceiver that reports the number of files matched by the receiver + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [24651] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/filestatsreceiver/documentation.md b/receiver/filestatsreceiver/documentation.md index 18921b2e6357..3d121aaba8d6 100644 --- a/receiver/filestatsreceiver/documentation.md +++ b/receiver/filestatsreceiver/documentation.md @@ -46,6 +46,14 @@ Elapsed time since last access of the file or folder, in seconds since Epoch. | ---- | ----------- | ---------- | ----------------------- | --------- | | s | Sum | Int | Cumulative | false | +### file.count + +The number of files matched + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {file} | Gauge | Int | + ### file.ctime Elapsed time since the last change of the file or folder, in seconds since Epoch. In addition to `file.mtime`, this metric tracks metadata changes such as permissions or renaming the file. diff --git a/receiver/filestatsreceiver/internal/metadata/generated_config.go b/receiver/filestatsreceiver/internal/metadata/generated_config.go index 5801768b284b..e90440d72786 100644 --- a/receiver/filestatsreceiver/internal/metadata/generated_config.go +++ b/receiver/filestatsreceiver/internal/metadata/generated_config.go @@ -26,6 +26,7 @@ func (ms *MetricConfig) Unmarshal(parser *confmap.Conf) error { // MetricsConfig provides config for filestats metrics. type MetricsConfig struct { FileAtime MetricConfig `mapstructure:"file.atime"` + FileCount MetricConfig `mapstructure:"file.count"` FileCtime MetricConfig `mapstructure:"file.ctime"` FileMtime MetricConfig `mapstructure:"file.mtime"` FileSize MetricConfig `mapstructure:"file.size"` @@ -36,6 +37,9 @@ func DefaultMetricsConfig() MetricsConfig { FileAtime: MetricConfig{ Enabled: false, }, + FileCount: MetricConfig{ + Enabled: false, + }, FileCtime: MetricConfig{ Enabled: false, }, diff --git a/receiver/filestatsreceiver/internal/metadata/generated_config_test.go b/receiver/filestatsreceiver/internal/metadata/generated_config_test.go index ec3952ce3ba1..225d0afecd2f 100644 --- a/receiver/filestatsreceiver/internal/metadata/generated_config_test.go +++ b/receiver/filestatsreceiver/internal/metadata/generated_config_test.go @@ -27,6 +27,7 @@ func TestMetricsBuilderConfig(t *testing.T) { want: MetricsBuilderConfig{ Metrics: MetricsConfig{ FileAtime: MetricConfig{Enabled: true}, + FileCount: MetricConfig{Enabled: true}, FileCtime: MetricConfig{Enabled: true}, FileMtime: MetricConfig{Enabled: true}, FileSize: MetricConfig{Enabled: true}, @@ -42,6 +43,7 @@ func TestMetricsBuilderConfig(t *testing.T) { want: MetricsBuilderConfig{ Metrics: MetricsConfig{ FileAtime: MetricConfig{Enabled: false}, + FileCount: MetricConfig{Enabled: false}, FileCtime: MetricConfig{Enabled: false}, FileMtime: MetricConfig{Enabled: false}, FileSize: MetricConfig{Enabled: false}, diff --git a/receiver/filestatsreceiver/internal/metadata/generated_metrics.go b/receiver/filestatsreceiver/internal/metadata/generated_metrics.go index d8e79116c6f6..498d4b99cb9f 100644 --- a/receiver/filestatsreceiver/internal/metadata/generated_metrics.go +++ b/receiver/filestatsreceiver/internal/metadata/generated_metrics.go @@ -62,6 +62,55 @@ func newMetricFileAtime(cfg MetricConfig) metricFileAtime { return m } +type metricFileCount struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills file.count metric with initial data. +func (m *metricFileCount) init() { + m.data.SetName("file.count") + m.data.SetDescription("The number of files matched") + m.data.SetUnit("{file}") + m.data.SetEmptyGauge() +} + +func (m *metricFileCount) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricFileCount) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricFileCount) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricFileCount(cfg MetricConfig) metricFileCount { + m := metricFileCount{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricFileCtime struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -224,6 +273,7 @@ type MetricsBuilder struct { metricsBuffer pmetric.Metrics // accumulates metrics data before emitting. buildInfo component.BuildInfo // contains version information. metricFileAtime metricFileAtime + metricFileCount metricFileCount metricFileCtime metricFileCtime metricFileMtime metricFileMtime metricFileSize metricFileSize @@ -246,6 +296,7 @@ func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.CreateSetting metricsBuffer: pmetric.NewMetrics(), buildInfo: settings.BuildInfo, metricFileAtime: newMetricFileAtime(mbc.Metrics.FileAtime), + metricFileCount: newMetricFileCount(mbc.Metrics.FileCount), metricFileCtime: newMetricFileCtime(mbc.Metrics.FileCtime), metricFileMtime: newMetricFileMtime(mbc.Metrics.FileMtime), metricFileSize: newMetricFileSize(mbc.Metrics.FileSize), @@ -311,6 +362,7 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { ils.Scope().SetVersion(mb.buildInfo.Version) ils.Metrics().EnsureCapacity(mb.metricsCapacity) mb.metricFileAtime.emit(ils.Metrics()) + mb.metricFileCount.emit(ils.Metrics()) mb.metricFileCtime.emit(ils.Metrics()) mb.metricFileMtime.emit(ils.Metrics()) mb.metricFileSize.emit(ils.Metrics()) @@ -339,6 +391,11 @@ func (mb *MetricsBuilder) RecordFileAtimeDataPoint(ts pcommon.Timestamp, val int mb.metricFileAtime.recordDataPoint(mb.startTime, ts, val) } +// RecordFileCountDataPoint adds a data point to file.count metric. +func (mb *MetricsBuilder) RecordFileCountDataPoint(ts pcommon.Timestamp, val int64) { + mb.metricFileCount.recordDataPoint(mb.startTime, ts, val) +} + // RecordFileCtimeDataPoint adds a data point to file.ctime metric. func (mb *MetricsBuilder) RecordFileCtimeDataPoint(ts pcommon.Timestamp, val int64, filePermissionsAttributeValue string) { mb.metricFileCtime.recordDataPoint(mb.startTime, ts, val, filePermissionsAttributeValue) diff --git a/receiver/filestatsreceiver/internal/metadata/generated_metrics_test.go b/receiver/filestatsreceiver/internal/metadata/generated_metrics_test.go index 22d0824c93a9..064d87407f16 100644 --- a/receiver/filestatsreceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/filestatsreceiver/internal/metadata/generated_metrics_test.go @@ -58,6 +58,9 @@ func TestMetricsBuilder(t *testing.T) { allMetricsCount++ mb.RecordFileAtimeDataPoint(ts, 1) + allMetricsCount++ + mb.RecordFileCountDataPoint(ts, 1) + allMetricsCount++ mb.RecordFileCtimeDataPoint(ts, 1, "file.permissions-val") @@ -108,6 +111,18 @@ func TestMetricsBuilder(t *testing.T) { assert.Equal(t, ts, dp.Timestamp()) assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) assert.Equal(t, int64(1), dp.IntValue()) + case "file.count": + assert.False(t, validatedMetrics["file.count"], "Found a duplicate in the metrics slice: file.count") + validatedMetrics["file.count"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "The number of files matched", ms.At(i).Description()) + assert.Equal(t, "{file}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) case "file.ctime": assert.False(t, validatedMetrics["file.ctime"], "Found a duplicate in the metrics slice: file.ctime") validatedMetrics["file.ctime"] = true diff --git a/receiver/filestatsreceiver/internal/metadata/testdata/config.yaml b/receiver/filestatsreceiver/internal/metadata/testdata/config.yaml index c5dc5ffc5d4a..dd4ee0b13247 100644 --- a/receiver/filestatsreceiver/internal/metadata/testdata/config.yaml +++ b/receiver/filestatsreceiver/internal/metadata/testdata/config.yaml @@ -3,6 +3,8 @@ all_set: metrics: file.atime: enabled: true + file.count: + enabled: true file.ctime: enabled: true file.mtime: @@ -18,6 +20,8 @@ none_set: metrics: file.atime: enabled: false + file.count: + enabled: false file.ctime: enabled: false file.mtime: diff --git a/receiver/filestatsreceiver/metadata.yaml b/receiver/filestatsreceiver/metadata.yaml index 1ad23d35a17c..eb24d6acae28 100644 --- a/receiver/filestatsreceiver/metadata.yaml +++ b/receiver/filestatsreceiver/metadata.yaml @@ -57,5 +57,11 @@ metrics: gauge: value_type: int unit: "b" + file.count: + description: The number of files matched + enabled: false + gauge: + value_type: int + unit: "{file}" tests: config: \ No newline at end of file diff --git a/receiver/filestatsreceiver/scraper.go b/receiver/filestatsreceiver/scraper.go index 065f0989edfe..5e09b5891947 100644 --- a/receiver/filestatsreceiver/scraper.go +++ b/receiver/filestatsreceiver/scraper.go @@ -57,6 +57,9 @@ func (s *scraper) scrape(_ context.Context) (pmetric.Metrics, error) { s.mb.EmitForResource(metadata.WithResource(rb.Emit())) } + s.mb.RecordFileCountDataPoint(now, int64(len(matches))) + s.mb.EmitForResource() + if len(scrapeErrors) > 0 { return s.mb.Emit(), scrapererror.NewPartialScrapeError(multierr.Combine(scrapeErrors...), len(scrapeErrors)) } diff --git a/receiver/filestatsreceiver/scraper_test.go b/receiver/filestatsreceiver/scraper_test.go index 0390a73866b5..4924f4da68cb 100644 --- a/receiver/filestatsreceiver/scraper_test.go +++ b/receiver/filestatsreceiver/scraper_test.go @@ -52,11 +52,15 @@ func Test_Scrape_All(t *testing.T) { cfg.Include = filepath.Join(tmpDir, "*.log") cfg.Metrics.FileAtime.Enabled = true cfg.Metrics.FileCtime.Enabled = true + cfg.Metrics.FileCount.Enabled = true s := newScraper(cfg, receivertest.NewNopCreateSettings()) metrics, err := s.scrape(context.Background()) require.NoError(t, err) - require.Equal(t, 0, metrics.ResourceMetrics().Len()) + require.Equal(t, 1, metrics.ResourceMetrics().Len()) + fileCount := metrics.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0) + require.Equal(t, int64(0), fileCount.Gauge().DataPoints().At(0).IntValue()) + require.Equal(t, "file.count", fileCount.Name()) logFile := filepath.Join(tmpDir, "my.log") err = os.WriteFile(logFile, []byte("something"), 0600) t.Cleanup(func() { @@ -71,8 +75,9 @@ func Test_Scrape_All(t *testing.T) { require.Equal(t, []string{logFile}, matches) metrics, err = s.scrape(context.Background()) require.NoError(t, err) - require.Equal(t, 1, metrics.ResourceMetrics().Len()) + require.Equal(t, 2, metrics.ResourceMetrics().Len()) require.Equal(t, 4, metrics.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().Len()) + require.Equal(t, 1, metrics.ResourceMetrics().At(1).ScopeMetrics().At(0).Metrics().Len()) aTimeMetric := metrics.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0) require.Equal(t, "file.atime", aTimeMetric.Name()) require.Equal(t, fileinfo.ModTime().Unix(), aTimeMetric.Sum().DataPoints().At(0).IntValue()) @@ -85,4 +90,7 @@ func Test_Scrape_All(t *testing.T) { sizeMetric := metrics.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3) require.Equal(t, "file.size", sizeMetric.Name()) require.Equal(t, int64(9), sizeMetric.Gauge().DataPoints().At(0).IntValue()) + fileCount = metrics.ResourceMetrics().At(1).ScopeMetrics().At(0).Metrics().At(0) + require.Equal(t, "file.count", fileCount.Name()) + require.Equal(t, int64(1), fileCount.Gauge().DataPoints().At(0).IntValue()) }