diff --git a/.chloggen/elasticsearchexporter-rm-dedup-config.yaml b/.chloggen/elasticsearchexporter-rm-dedup-config.yaml new file mode 100644 index 0000000000000..7b3ab00d9c53a --- /dev/null +++ b/.chloggen/elasticsearchexporter-rm-dedup-config.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove `dedup` config setting that was deprecated in v0.104.0 + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33773] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/.chloggen/logdedupprocessor-config-validate.yaml b/.chloggen/logdedupprocessor-config-validate.yaml new file mode 100644 index 0000000000000..dcd01f95ff2df --- /dev/null +++ b/.chloggen/logdedupprocessor-config-validate.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: logdedupprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Fix config validation not working when creating a processor. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37278] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index af0af6faf2a21..aa8bb6518ccd8 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -165,9 +165,6 @@ behaviours, which may be configured through the following settings: It works only for logs where the log record body is a map. Each LogRecord body is serialized to JSON as-is and becomes a separate document for ingestion. If the log record body is not a map, the exporter will log a warning and drop the log record. - - `dedup` (DEPRECATED). This configuration is deprecated and non-operational, - and will be removed in the future. Object keys are always deduplicated to - avoid Elasticsearch rejecting documents. - `dedot` (default=true; DEPRECATED, in future dedotting will always be enabled for ECS mode, and never for other modes): When enabled attributes with `.` will be split into proper json objects. diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 12dc5651204a3..40acdaf99497c 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -183,13 +183,6 @@ type MappingsSettings struct { // Mode configures the field mappings. Mode string `mapstructure:"mode"` - // Dedup is non-operational, and will be removed in the future. - // - // Deprecated: [v0.104.0] deduplication is always enabled, and cannot be - // disabled. Disabling deduplication is not meaningful, as Elasticsearch - // will always reject documents with duplicate JSON object keys. - Dedup *bool `mapstructure:"dedup,omitempty"` - // Deprecated: [v0.104.0] dedotting will always be applied for ECS mode // in future, and never for other modes. Elasticsearch's "dot_expander" // Ingest processor may be used as an alternative for non-ECS modes. @@ -363,9 +356,6 @@ func (cfg *Config) MappingMode() MappingMode { } func handleDeprecatedConfig(cfg *Config, logger *zap.Logger) { - if cfg.Mapping.Dedup != nil { - logger.Warn("dedup is deprecated, and is always enabled") - } if cfg.Mapping.Dedot && cfg.MappingMode() != MappingECS || !cfg.Mapping.Dedot && cfg.MappingMode() == MappingECS { logger.Warn("dedot has been deprecated: in the future, dedotting will always be performed in ECS mode only") } diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 6125988ea4631..3b0490b151722 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -240,7 +240,7 @@ func TestExporterLogs(t *testing.T) { exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { cfg.Mapping.Mode = "raw" - // dedup is the default + // deduplication is always performed - there is no configuration that controls it }) logs := newLogsWithAttributes( // Scope collides with the top-level "Scope" field, diff --git a/exporter/elasticsearchexporter/factory_test.go b/exporter/elasticsearchexporter/factory_test.go index 80f294f377057..7a36b87658b8b 100644 --- a/exporter/elasticsearchexporter/factory_test.go +++ b/exporter/elasticsearchexporter/factory_test.go @@ -61,38 +61,6 @@ func TestFactory_CreateTraces(t *testing.T) { require.NoError(t, exporter.Shutdown(context.Background())) } -func TestFactory_DedupDeprecated(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - dedup := false - cfg.Endpoint = "http://testing.invalid:9200" - cfg.Mapping.Dedup = &dedup - cfg.Mapping.Dedot = false // avoid dedot warnings - }) - - loggerCore, logObserver := observer.New(zap.WarnLevel) - set := exportertest.NewNopSettings() - set.Logger = zap.New(loggerCore) - - logsExporter, err := factory.CreateLogs(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, logsExporter.Shutdown(context.Background())) - - tracesExporter, err := factory.CreateTraces(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, tracesExporter.Shutdown(context.Background())) - - metricsExporter, err := factory.CreateMetrics(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, metricsExporter.Shutdown(context.Background())) - - records := logObserver.AllUntimed() - assert.Len(t, records, 3) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[0].Message) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[1].Message) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[2].Message) -} - func TestFactory_DedotDeprecated(t *testing.T) { loggerCore, logObserver := observer.New(zap.WarnLevel) set := exportertest.NewNopSettings() diff --git a/processor/logdedupprocessor/factory.go b/processor/logdedupprocessor/factory.go index 0931e06cc176d..c2c0f5e7f66b5 100644 --- a/processor/logdedupprocessor/factory.go +++ b/processor/logdedupprocessor/factory.go @@ -32,6 +32,10 @@ func createLogsProcessor(_ context.Context, settings processor.Settings, cfg com return nil, fmt.Errorf("invalid config type: %+v", cfg) } + if err := processorCfg.Validate(); err != nil { + return nil, err + } + processor, err := newProcessor(processorCfg, consumer, settings) if err != nil { return nil, fmt.Errorf("error creating processor: %w", err) diff --git a/processor/logdedupprocessor/processor_test.go b/processor/logdedupprocessor/processor_test.go index 72aa5919aad7e..69e953caf5376 100644 --- a/processor/logdedupprocessor/processor_test.go +++ b/processor/logdedupprocessor/processor_test.go @@ -308,3 +308,24 @@ func TestProcessorConsumeMultipleConditions(t *testing.T) { err = p.Shutdown(context.Background()) require.NoError(t, err) } + +func TestProcessorConfigValidate(t *testing.T) { + t.Parallel() + invalidCfg := &Config{ + LogCountAttribute: defaultLogCountAttribute, + Interval: -1, + Timezone: "", + } + + _, err := createLogsProcessor(context.Background(), processortest.NewNopSettings(), invalidCfg, consumertest.NewNop()) + require.Error(t, err) + + validCfg := &Config{ + LogCountAttribute: defaultLogCountAttribute, + Interval: defaultInterval, + Timezone: defaultTimezone, + } + + _, err = createLogsProcessor(context.Background(), processortest.NewNopSettings(), validCfg, consumertest.NewNop()) + require.NoError(t, err) +}