Merge pull request #3 from scalyr/dataset_log_exporter_improvements

[DSET-4037] [DSET-4014] [DSET-4034] [DSET-4034] [DSET-4098] Dataset log exporter improvements

.chloggen/dataset-exporter-various-improvements-and-fixes.yaml

@@ -0,0 +1,20 @@
+# Use this changelog template to create an entry for release notes.
+# If your change doesn't affect end users, such as a test fix or a tooling change,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/datasetexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Correctly map LogRecord severity to DataSet severity, remove redundant DataSet event message field prefix (OtelExporter - Log -) and remove redundant DataSet event fields (flags, flags.is_sampled)."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [20660]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

exporter/datasetexporter/README.md

@@ -4,9 +4,10 @@
 | Status        |           |
 | ------------- |-----------|
 | Stability     | [alpha]: logs, traces   |
-| Distributions | [] |
+| Distributions | [contrib] |
 
 [alpha]: https://github.com/open-telemetry/opentelemetry-collector#alpha
+[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
 <!-- end autogenerated section -->
 
 This exporter sends logs to [DataSet](https://www.dataset.com/).

exporter/datasetexporter/logs_exporter.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/scalyr/dataset-go/pkg/api/add_events"
@@ -19,6 +20,21 @@ import (
 
 var now = time.Now
 
+// If a LogRecord doesn't contain severity or we can't map it to a valid DataSet severity, we use
+// this value (3 - INFO) instead
+const defaultDataSetSeverityLevel int = dataSetLogLevelInfo
+
+// Constants for valid DataSet log levels (aka Event.sev int field value)
+const (
+	dataSetLogLevelFinest = 0
+	dataSetLogLevelTrace  = 1
+	dataSetLogLevelDebug  = 2
+	dataSetLogLevelInfo   = 3
+	dataSetLogLevelWarn   = 4
+	dataSetLogLevelError  = 5
+	dataSetLogLevelFatal  = 6
+)
+
 func createLogsExporter(ctx context.Context, set exporter.CreateSettings, config component.Config) (exporter.Logs, error) {
 	cfg := castConfig(config)
 	e, err := newDatasetExporter("logs", cfg, set.Logger)
@@ -65,6 +81,95 @@ func buildBody(attrs map[string]interface{}, value pcommon.Value) string {
 	return message
 }
 
+// Function maps OTel severity on the LogRecord to DataSet severity level (number)
+func mapOtelSeverityToDataSetSeverity(log plog.LogRecord) int {
+	// This function maps OTel severity level to DataSet severity levels
+	//
+	// Valid OTel levels - https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+	// and valid DataSet ones - https://github.com/scalyr/logstash-output-scalyr/blob/master/lib/logstash/outputs/scalyr.rb#L70
+	sevNum := log.SeverityNumber()
+	sevText := log.SeverityText()
+
+	dataSetSeverity := defaultDataSetSeverityLevel
+
+	if sevNum > 0 {
+		dataSetSeverity = mapLogRecordSevNumToDataSetSeverity(sevNum)
+	} else if sevText != "" {
+		// Per docs, SeverityNumber is optional so if it's not present we fall back to SeverityText
+		// https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext
+		dataSetSeverity = mapLogRecordSeverityTextToDataSetSeverity(sevText)
+	}
+
+	// TODO: We should log in case we see invalid severity, but right now, afaik, we / OTEL
+	// don't have a concept of "rate limited" logging. We don't want to log every single
+	// occurrence in case there are many log records like that since this could cause a lot of
+	// noise and performance overhead
+
+	return dataSetSeverity
+}
+
+func mapLogRecordSevNumToDataSetSeverity(sevNum plog.SeverityNumber) int {
+	// Maps LogRecord.SeverityNumber field value to DataSet severity value.
+	dataSetSeverity := defaultDataSetSeverityLevel
+
+	if sevNum <= 0 {
+		return dataSetSeverity
+	}
+
+	// See https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+	// for OTEL mappings
+	switch sevNum {
+	case 1, 2, 3, 4:
+		// TRACE
+		dataSetSeverity = dataSetLogLevelTrace
+	case 5, 6, 7, 8:
+		// DEBUG
+		dataSetSeverity = dataSetLogLevelDebug
+	case 9, 10, 11, 12:
+		// INFO
+		dataSetSeverity = dataSetLogLevelInfo
+	case 13, 14, 15, 16:
+		// WARN
+		dataSetSeverity = dataSetLogLevelWarn
+	case 17, 18, 19, 20:
+		// ERROR
+		dataSetSeverity = dataSetLogLevelError
+	case 21, 22, 23, 24:
+		// FATAL / CRITICAL / EMERGENCY
+		dataSetSeverity = dataSetLogLevelFatal
+	}
+
+	return dataSetSeverity
+}
+
+func mapLogRecordSeverityTextToDataSetSeverity(sevText string) int {
+	// Maps LogRecord.SeverityText field value to DataSet severity value.
+	dataSetSeverity := defaultDataSetSeverityLevel
+
+	if sevText == "" {
+		return dataSetSeverity
+	}
+
+	switch strings.ToLower(sevText) {
+	case "fine", "finest":
+		dataSetSeverity = dataSetLogLevelFinest
+	case "trace":
+		dataSetSeverity = dataSetLogLevelTrace
+	case "debug":
+		dataSetSeverity = dataSetLogLevelDebug
+	case "info", "information":
+		dataSetSeverity = dataSetLogLevelInfo
+	case "warn", "warning":
+		dataSetSeverity = dataSetLogLevelWarn
+	case "error":
+		dataSetSeverity = dataSetLogLevelError
+	case "fatal", "critical", "emergency":
+		dataSetSeverity = dataSetLogLevelFatal
+	}
+
+	return dataSetSeverity
+}
+
 func buildEventFromLog(
 	log plog.LogRecord,
 	resource pcommon.Resource,
@@ -75,30 +180,22 @@ func buildEventFromLog(
 	event := add_events.Event{}
 
 	observedTs := log.ObservedTimestamp().AsTime()
-	if sevNum := log.SeverityNumber(); sevNum > 0 {
-		attrs["severity.number"] = sevNum
-		event.Sev = int(sevNum)
-	}
+
+	event.Sev = mapOtelSeverityToDataSetSeverity(log)
 
 	if timestamp := log.Timestamp().AsTime(); !timestamp.Equal(time.Unix(0, 0)) {
 		event.Ts = strconv.FormatInt(timestamp.UnixNano(), 10)
 	}
 
 	if body := log.Body().AsString(); body != "" {
-		attrs["message"] = fmt.Sprintf(
-			"OtelExporter - Log - %s",
-			buildBody(attrs, log.Body()),
-		)
+		attrs["message"] = buildBody(attrs, log.Body())
 	}
 	if dropped := log.DroppedAttributesCount(); dropped > 0 {
 		attrs["dropped_attributes_count"] = dropped
 	}
 	if !observedTs.Equal(time.Unix(0, 0)) {
 		attrs["observed.timestamp"] = observedTs.String()
 	}
-	if sevText := log.SeverityText(); sevText != "" {
-		attrs["severity.text"] = sevText
-	}
 	if span := log.SpanID().String(); span != "" {
 		attrs["span_id"] = span
 	}
@@ -122,8 +219,6 @@ func buildEventFromLog(
 	}
 
 	updateWithPrefixedValues(attrs, "attributes.", ".", log.Attributes().AsRaw(), 0)
-	attrs["flags"] = log.Flags()
-	attrs["flag.is_sampled"] = log.Flags().IsSampled()
 
 	if settings.ExportResourceInfo {
 		updateWithPrefixedValues(attrs, "resource.attributes.", ".", resource.Attributes().AsRaw(), 0)