Merge branch 'main' into chan-tim_mongodbMetrics

.chloggen/elasticsearchexporter-rm-dedup-config.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Remove `dedup` config setting that was deprecated in v0.104.0
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33773]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.chloggen/logdedupprocessor-config-validate.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: logdedupprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fix config validation not working when creating a processor.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [37278]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/elasticsearchexporter/README.md

@@ -165,9 +165,6 @@ behaviours, which may be configured through the following settings:
             It works only for logs where the log record body is a map. Each LogRecord
             body is serialized to JSON as-is and becomes a separate document for ingestion.
             If the log record body is not a map, the exporter will log a warning and drop the log record.
-  - `dedup` (DEPRECATED). This configuration is deprecated and non-operational,
-    and will be removed in the future. Object keys are always deduplicated to
-    avoid Elasticsearch rejecting documents.
   - `dedot` (default=true; DEPRECATED, in future dedotting will always be enabled
     for ECS mode, and never for other modes): When enabled attributes with `.`
     will be split into proper json objects.

exporter/elasticsearchexporter/config.go

@@ -183,13 +183,6 @@ type MappingsSettings struct {
 	// Mode configures the field mappings.
 	Mode string `mapstructure:"mode"`
 
-	// Dedup is non-operational, and will be removed in the future.
-	//
-	// Deprecated: [v0.104.0] deduplication is always enabled, and cannot be
-	// disabled. Disabling deduplication is not meaningful, as Elasticsearch
-	// will always reject documents with duplicate JSON object keys.
-	Dedup *bool `mapstructure:"dedup,omitempty"`
-
 	// Deprecated: [v0.104.0] dedotting will always be applied for ECS mode
 	// in future, and never for other modes. Elasticsearch's "dot_expander"
 	// Ingest processor may be used as an alternative for non-ECS modes.
@@ -363,9 +356,6 @@ func (cfg *Config) MappingMode() MappingMode {
 }
 
 func handleDeprecatedConfig(cfg *Config, logger *zap.Logger) {
-	if cfg.Mapping.Dedup != nil {
-		logger.Warn("dedup is deprecated, and is always enabled")
-	}
 	if cfg.Mapping.Dedot && cfg.MappingMode() != MappingECS || !cfg.Mapping.Dedot && cfg.MappingMode() == MappingECS {
 		logger.Warn("dedot has been deprecated: in the future, dedotting will always be performed in ECS mode only")
 	}

exporter/elasticsearchexporter/exporter_test.go

@@ -240,7 +240,7 @@ func TestExporterLogs(t *testing.T) {
 
 		exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) {
 			cfg.Mapping.Mode = "raw"
-			// dedup is the default
+			// deduplication is always performed - there is no configuration that controls it
 		})
 		logs := newLogsWithAttributes(
 			// Scope collides with the top-level "Scope" field,

exporter/elasticsearchexporter/factory_test.go

@@ -61,38 +61,6 @@ func TestFactory_CreateTraces(t *testing.T) {
 	require.NoError(t, exporter.Shutdown(context.Background()))
 }
 
-func TestFactory_DedupDeprecated(t *testing.T) {
-	factory := NewFactory()
-	cfg := withDefaultConfig(func(cfg *Config) {
-		dedup := false
-		cfg.Endpoint = "http://testing.invalid:9200"
-		cfg.Mapping.Dedup = &dedup
-		cfg.Mapping.Dedot = false // avoid dedot warnings
-	})
-
-	loggerCore, logObserver := observer.New(zap.WarnLevel)
-	set := exportertest.NewNopSettings()
-	set.Logger = zap.New(loggerCore)
-
-	logsExporter, err := factory.CreateLogs(context.Background(), set, cfg)
-	require.NoError(t, err)
-	require.NoError(t, logsExporter.Shutdown(context.Background()))
-
-	tracesExporter, err := factory.CreateTraces(context.Background(), set, cfg)
-	require.NoError(t, err)
-	require.NoError(t, tracesExporter.Shutdown(context.Background()))
-
-	metricsExporter, err := factory.CreateMetrics(context.Background(), set, cfg)
-	require.NoError(t, err)
-	require.NoError(t, metricsExporter.Shutdown(context.Background()))
-
-	records := logObserver.AllUntimed()
-	assert.Len(t, records, 3)
-	assert.Equal(t, "dedup is deprecated, and is always enabled", records[0].Message)
-	assert.Equal(t, "dedup is deprecated, and is always enabled", records[1].Message)
-	assert.Equal(t, "dedup is deprecated, and is always enabled", records[2].Message)
-}
-
 func TestFactory_DedotDeprecated(t *testing.T) {
 	loggerCore, logObserver := observer.New(zap.WarnLevel)
 	set := exportertest.NewNopSettings()

processor/logdedupprocessor/factory.go

@@ -32,6 +32,10 @@ func createLogsProcessor(_ context.Context, settings processor.Settings, cfg com
 		return nil, fmt.Errorf("invalid config type: %+v", cfg)
 	}
 
+	if err := processorCfg.Validate(); err != nil {
+		return nil, err
+	}
+
 	processor, err := newProcessor(processorCfg, consumer, settings)
 	if err != nil {
 		return nil, fmt.Errorf("error creating processor: %w", err)

processor/logdedupprocessor/processor_test.go

@@ -308,3 +308,24 @@ func TestProcessorConsumeMultipleConditions(t *testing.T) {
 	err = p.Shutdown(context.Background())
 	require.NoError(t, err)
 }
+
+func TestProcessorConfigValidate(t *testing.T) {
+	t.Parallel()
+	invalidCfg := &Config{
+		LogCountAttribute: defaultLogCountAttribute,
+		Interval:          -1,
+		Timezone:          "",
+	}
+
+	_, err := createLogsProcessor(context.Background(), processortest.NewNopSettings(), invalidCfg, consumertest.NewNop())
+	require.Error(t, err)
+
+	validCfg := &Config{
+		LogCountAttribute: defaultLogCountAttribute,
+		Interval:          defaultInterval,
+		Timezone:          defaultTimezone,
+	}
+
+	_, err = createLogsProcessor(context.Background(), processortest.NewNopSettings(), validCfg, consumertest.NewNop())
+	require.NoError(t, err)
+}