diff --git a/.chloggen/dataset-exporter-various-improvements-and-fixes.yaml b/.chloggen/dataset-exporter-various-improvements-and-fixes.yaml new file mode 100644 index 000000000000..62207ac59d7d --- /dev/null +++ b/.chloggen/dataset-exporter-various-improvements-and-fixes.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/datasetexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Correctly map LogRecord severity to DataSet severity, remove redundant DataSet event message field prefix (OtelExporter - Log -) and remove redundant DataSet event fields (flags, flags.is_sampled)." + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [20660] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/exporter/datasetexporter/README.md b/exporter/datasetexporter/README.md index ceef4497e2ea..dcf867a07713 100644 --- a/exporter/datasetexporter/README.md +++ b/exporter/datasetexporter/README.md @@ -4,9 +4,10 @@ | Status | | | ------------- |-----------| | Stability | [alpha]: logs, traces | -| Distributions | [] | +| Distributions | [contrib] | [alpha]: https://github.com/open-telemetry/opentelemetry-collector#alpha +[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib This exporter sends logs to [DataSet](https://www.dataset.com/). diff --git a/exporter/datasetexporter/logs_exporter.go b/exporter/datasetexporter/logs_exporter.go index 2aed24b9d027..f3b3a379ed1b 100644 --- a/exporter/datasetexporter/logs_exporter.go +++ b/exporter/datasetexporter/logs_exporter.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/scalyr/dataset-go/pkg/api/add_events" @@ -19,6 +20,21 @@ import ( var now = time.Now +// If a LogRecord doesn't contain severity or we can't map it to a valid DataSet severity, we use +// this value (3 - INFO) instead +const defaultDataSetSeverityLevel int = dataSetLogLevelInfo + +// Constants for valid DataSet log levels (aka Event.sev int field value) +const ( + dataSetLogLevelFinest = 0 + dataSetLogLevelTrace = 1 + dataSetLogLevelDebug = 2 + dataSetLogLevelInfo = 3 + dataSetLogLevelWarn = 4 + dataSetLogLevelError = 5 + dataSetLogLevelFatal = 6 +) + func createLogsExporter(ctx context.Context, set exporter.CreateSettings, config component.Config) (exporter.Logs, error) { cfg := castConfig(config) e, err := newDatasetExporter("logs", cfg, set.Logger) @@ -65,6 +81,95 @@ func buildBody(attrs map[string]interface{}, value pcommon.Value) string { return message } +// Function maps OTel severity on the LogRecord to DataSet severity level (number) +func mapOtelSeverityToDataSetSeverity(log plog.LogRecord) int { + // This function maps OTel severity level to DataSet severity levels + // + // Valid OTel levels - https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber + // and valid DataSet ones - https://github.com/scalyr/logstash-output-scalyr/blob/master/lib/logstash/outputs/scalyr.rb#L70 + sevNum := log.SeverityNumber() + sevText := log.SeverityText() + + dataSetSeverity := defaultDataSetSeverityLevel + + if sevNum > 0 { + dataSetSeverity = mapLogRecordSevNumToDataSetSeverity(sevNum) + } else if sevText != "" { + // Per docs, SeverityNumber is optional so if it's not present we fall back to SeverityText + // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext + dataSetSeverity = mapLogRecordSeverityTextToDataSetSeverity(sevText) + } + + // TODO: We should log in case we see invalid severity, but right now, afaik, we / OTEL + // don't have a concept of "rate limited" logging. We don't want to log every single + // occurrence in case there are many log records like that since this could cause a lot of + // noise and performance overhead + + return dataSetSeverity +} + +func mapLogRecordSevNumToDataSetSeverity(sevNum plog.SeverityNumber) int { + // Maps LogRecord.SeverityNumber field value to DataSet severity value. + dataSetSeverity := defaultDataSetSeverityLevel + + if sevNum <= 0 { + return dataSetSeverity + } + + // See https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber + // for OTEL mappings + switch sevNum { + case 1, 2, 3, 4: + // TRACE + dataSetSeverity = dataSetLogLevelTrace + case 5, 6, 7, 8: + // DEBUG + dataSetSeverity = dataSetLogLevelDebug + case 9, 10, 11, 12: + // INFO + dataSetSeverity = dataSetLogLevelInfo + case 13, 14, 15, 16: + // WARN + dataSetSeverity = dataSetLogLevelWarn + case 17, 18, 19, 20: + // ERROR + dataSetSeverity = dataSetLogLevelError + case 21, 22, 23, 24: + // FATAL / CRITICAL / EMERGENCY + dataSetSeverity = dataSetLogLevelFatal + } + + return dataSetSeverity +} + +func mapLogRecordSeverityTextToDataSetSeverity(sevText string) int { + // Maps LogRecord.SeverityText field value to DataSet severity value. + dataSetSeverity := defaultDataSetSeverityLevel + + if sevText == "" { + return dataSetSeverity + } + + switch strings.ToLower(sevText) { + case "fine", "finest": + dataSetSeverity = dataSetLogLevelFinest + case "trace": + dataSetSeverity = dataSetLogLevelTrace + case "debug": + dataSetSeverity = dataSetLogLevelDebug + case "info", "information": + dataSetSeverity = dataSetLogLevelInfo + case "warn", "warning": + dataSetSeverity = dataSetLogLevelWarn + case "error": + dataSetSeverity = dataSetLogLevelError + case "fatal", "critical", "emergency": + dataSetSeverity = dataSetLogLevelFatal + } + + return dataSetSeverity +} + func buildEventFromLog( log plog.LogRecord, resource pcommon.Resource, @@ -75,20 +180,15 @@ func buildEventFromLog( event := add_events.Event{} observedTs := log.ObservedTimestamp().AsTime() - if sevNum := log.SeverityNumber(); sevNum > 0 { - attrs["severity.number"] = sevNum - event.Sev = int(sevNum) - } + + event.Sev = mapOtelSeverityToDataSetSeverity(log) if timestamp := log.Timestamp().AsTime(); !timestamp.Equal(time.Unix(0, 0)) { event.Ts = strconv.FormatInt(timestamp.UnixNano(), 10) } if body := log.Body().AsString(); body != "" { - attrs["message"] = fmt.Sprintf( - "OtelExporter - Log - %s", - buildBody(attrs, log.Body()), - ) + attrs["message"] = buildBody(attrs, log.Body()) } if dropped := log.DroppedAttributesCount(); dropped > 0 { attrs["dropped_attributes_count"] = dropped @@ -96,9 +196,6 @@ func buildEventFromLog( if !observedTs.Equal(time.Unix(0, 0)) { attrs["observed.timestamp"] = observedTs.String() } - if sevText := log.SeverityText(); sevText != "" { - attrs["severity.text"] = sevText - } if span := log.SpanID().String(); span != "" { attrs["span_id"] = span } @@ -122,8 +219,6 @@ func buildEventFromLog( } updateWithPrefixedValues(attrs, "attributes.", ".", log.Attributes().AsRaw(), 0) - attrs["flags"] = log.Flags() - attrs["flag.is_sampled"] = log.Flags().IsSampled() if settings.ExportResourceInfo { updateWithPrefixedValues(attrs, "resource.attributes.", ".", resource.Attributes().AsRaw(), 0) diff --git a/exporter/datasetexporter/logs_exporter_test.go b/exporter/datasetexporter/logs_exporter_test.go index e61d42932b3a..92824824df10 100644 --- a/exporter/datasetexporter/logs_exporter_test.go +++ b/exporter/datasetexporter/logs_exporter_test.go @@ -156,7 +156,7 @@ func TestBuildBodyMap(t *testing.T) { var testLEventRaw = &add_events.Event{ Thread: "TL", Log: "LL", - Sev: 9, + Sev: 3, Ts: "1581452773000000789", Attrs: map[string]interface{}{ "attributes.app": "server", @@ -164,12 +164,8 @@ var testLEventRaw = &add_events.Event{ "body.str": "This is a log message", "body.type": "Str", "dropped_attributes_count": uint32(1), - "flag.is_sampled": false, - "flags": plog.LogRecordFlags(0), - "message": "OtelExporter - Log - This is a log message", + "message": "This is a log message", "scope.name": "", - "severity.number": plog.SeverityNumberInfo, - "severity.text": "Info", "span_id": "0102040800000000", "trace_id": "08040201000000000000000000000000", }, @@ -186,12 +182,8 @@ var testLEventReq = &add_events.Event{ "body.str": "This is a log message", "body.type": "Str", "dropped_attributes_count": float64(1), - "flag.is_sampled": false, - "flags": float64(plog.LogRecordFlags(0)), - "message": "OtelExporter - Log - This is a log message", + "message": "This is a log message", "scope.name": "", - "severity.number": float64(plog.SeverityNumberInfo), - "severity.text": "Info", "span_id": "0102040800000000", "trace_id": "08040201000000000000000000000000", "bundle_key": "d41d8cd98f00b204e9800998ecf8427e", @@ -407,3 +399,187 @@ func TestConsumeLogsShouldSucceed(t *testing.T) { addRequest, ) } + +func makeLogRecordWithSeverityNumberAndSeverityText(sevNum int, sevText string) plog.LogRecord { + lr := testdata.GenerateLogsOneLogRecord() + ld := lr.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0) + + ld.SetSeverityNumber(plog.SeverityNumber(sevNum)) + ld.SetSeverityText(sevText) + + return ld +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextInvalidValues(t *testing.T) { + // Invalid values get mapped to info (3 - INFO) + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "") + assert.Equal(t, defaultDataSetSeverityLevel, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(-1, "") + assert.Equal(t, defaultDataSetSeverityLevel, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(25, "") + assert.Equal(t, defaultDataSetSeverityLevel, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(100, "") + assert.Equal(t, defaultDataSetSeverityLevel, mapOtelSeverityToDataSetSeverity(ld)) + +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextDataSetTraceLogLevel(t *testing.T) { + // trace + ld := makeLogRecordWithSeverityNumberAndSeverityText(1, "") + assert.Equal(t, dataSetLogLevelTrace, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(2, "") + assert.Equal(t, dataSetLogLevelTrace, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(3, "") + assert.Equal(t, dataSetLogLevelTrace, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(4, "") + assert.Equal(t, dataSetLogLevelTrace, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextDataSetDebugLogLevel(t *testing.T) { + // debug + ld := makeLogRecordWithSeverityNumberAndSeverityText(5, "") + assert.Equal(t, dataSetLogLevelDebug, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(6, "") + assert.Equal(t, dataSetLogLevelDebug, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(7, "") + assert.Equal(t, dataSetLogLevelDebug, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(8, "") + assert.Equal(t, dataSetLogLevelDebug, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextDataSetInfoLogLevel(t *testing.T) { + // info + ld := makeLogRecordWithSeverityNumberAndSeverityText(9, "") + assert.Equal(t, dataSetLogLevelInfo, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(10, "") + assert.Equal(t, dataSetLogLevelInfo, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(11, "") + assert.Equal(t, dataSetLogLevelInfo, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(12, "") + assert.Equal(t, dataSetLogLevelInfo, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextDataSetWarnLogLevel(t *testing.T) { + // warn + ld := makeLogRecordWithSeverityNumberAndSeverityText(13, "") + assert.Equal(t, dataSetLogLevelWarn, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(14, "") + assert.Equal(t, dataSetLogLevelWarn, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(15, "") + assert.Equal(t, dataSetLogLevelWarn, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(16, "") + assert.Equal(t, dataSetLogLevelWarn, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextDataSetErrorLogLevel(t *testing.T) { + // error + ld := makeLogRecordWithSeverityNumberAndSeverityText(17, "") + assert.Equal(t, dataSetLogLevelError, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(18, "") + assert.Equal(t, dataSetLogLevelError, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(19, "") + assert.Equal(t, dataSetLogLevelError, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(20, "") + assert.Equal(t, dataSetLogLevelError, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityTextDataSetFatalLogLevel(t *testing.T) { + // fatal + ld := makeLogRecordWithSeverityNumberAndSeverityText(21, "") + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(22, "") + ld.SetSeverityNumber(22) + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(23, "") + ld.SetSeverityNumber(22) + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(24, "") + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberInvalidValues(t *testing.T) { + // invalid values + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "a") + assert.Equal(t, defaultDataSetSeverityLevel, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(0, "infoinfo") + assert.Equal(t, defaultDataSetSeverityLevel, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberDataSetTraceLogLevel(t *testing.T) { + // trace + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "trace") + assert.Equal(t, dataSetLogLevelTrace, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberDataSetDebugLogLevel(t *testing.T) { + // debug + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "debug") + assert.Equal(t, dataSetLogLevelDebug, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberDataSetInfoLogLevel(t *testing.T) { + // info + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "info") + assert.Equal(t, dataSetLogLevelInfo, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(0, "informational") + ld.SetSeverityText("informational") +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberDataSetInfoWarnLevel(t *testing.T) { + // warn + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "warn") + assert.Equal(t, dataSetLogLevelWarn, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(0, "warning") + assert.Equal(t, dataSetLogLevelWarn, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberDataSetInfoErrorLevel(t *testing.T) { + // error + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "error") + assert.Equal(t, dataSetLogLevelError, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumberDataSetInfoFatalLevel(t *testing.T) { + // fatal + ld := makeLogRecordWithSeverityNumberAndSeverityText(0, "fatal") + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(0, "fatal") + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(0, "emergency") + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) +} + +func TestOtelSeverityToDataSetSeverityWithSeverityNumberAndSeverityTextSeverityNumberHasPriority(t *testing.T) { + // If provided, SeverityNumber has priority over SeverityText + ld := makeLogRecordWithSeverityNumberAndSeverityText(3, "debug") + assert.Equal(t, dataSetLogLevelTrace, mapOtelSeverityToDataSetSeverity(ld)) + + ld = makeLogRecordWithSeverityNumberAndSeverityText(22, "info") + assert.Equal(t, dataSetLogLevelFatal, mapOtelSeverityToDataSetSeverity(ld)) +} diff --git a/exporter/datasetexporter/metadata.yaml b/exporter/datasetexporter/metadata.yaml index 89a9faf47ea4..b7abd4cf08a6 100644 --- a/exporter/datasetexporter/metadata.yaml +++ b/exporter/datasetexporter/metadata.yaml @@ -4,4 +4,4 @@ status: class: exporter stability: alpha: [logs, traces] - distributions: [] + distributions: [contrib]