[DSET-4037] [DSET-4014] [DSET-4034] [DSET-4034] [DSET-4098] Dataset log exporter improvements

exporter/datasetexporter/logs_exporter.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/scalyr/dataset-go/pkg/api/add_events"
@@ -65,6 +66,73 @@ func buildBody(attrs map[string]interface{}, value pcommon.Value) string {
 	return message
 }
 
+// Function maps OTel severity on the LogRecord to DataSet severity level (number)
+func otelSeverityToDataSetSeverity(log plog.LogRecord) int {
+	// If log record doesn't contain severity or we can't map it to a valid DataSet severity,
+	// we use this value (INFO) instead
+	defaultSeverityLevel := 3
+
+	// This function maps OTel severity level to DataSet severity levels
+	//
+	// Valid OTel levels - https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+	// and valid DataSet ones - https://github.com/scalyr/logstash-output-scalyr/blob/master/lib/logstash/outputs/scalyr.rb#L70
+	sevNum := log.SeverityNumber()
+	sevText := log.SeverityText()
+
+	dataSetSeverity := defaultSeverityLevel
+
+	if sevNum > 0 {
+		// See https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+		// for OTEL mappings
+		switch sevNum {
+		case 1, 2, 3, 4:
+			// TRACE
+			dataSetSeverity = 1
+		case 5, 6, 7, 8:
+			// DEBUG
+			dataSetSeverity = 2
+		case 9, 10, 11, 12:
+			// INFO
+			dataSetSeverity = 3
+		case 13, 14, 15, 16:
+			// WARN
+			dataSetSeverity = 4
+		case 17, 18, 19, 20:
+			// ERROR
+			dataSetSeverity = 5
+		case 21, 22, 23, 24:
+			// FATAL / CRITICAL / EMERGENCY
+			dataSetSeverity = 6
+		}
+	} else if sevText != "" {
+		// Per docs, SeverityNumber is optional so if it's not present we fall back to SeverityText
+		// https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext
+		switch strings.ToLower(sevText) {
+		case "fine", "finest":
+			dataSetSeverity = 0
+		case "trace":
+			dataSetSeverity = 1
+		case "debug":
+			dataSetSeverity = 2
+		case "info", "information":
+			dataSetSeverity = 3
+		case "warn", "warning":
+			dataSetSeverity = 4
+		case "error":
+			dataSetSeverity = 5
+		case "fatal", "critical", "emergency":
+			dataSetSeverity = 6
+		}
+	}
+
+	// TODO: We should log in case we see invalid severity, but right now, afaik, we / OTEL
+	// don't have a concept of "rate limited" logging. We don't want to log every single
+	// occurrence in case there are many log records like that since this could cause a lot of
+	// noise and performance overhead
+
+	return dataSetSeverity
+}
+
 func buildEventFromLog(
 	log plog.LogRecord,
 	resource pcommon.Resource,
@@ -75,10 +143,8 @@ func buildEventFromLog(
 	event := add_events.Event{}
 
 	observedTs := log.ObservedTimestamp().AsTime()
-	if sevNum := log.SeverityNumber(); sevNum > 0 {
-		attrs["severity.number"] = sevNum
-		event.Sev = int(sevNum)
-	}
+
+	event.Sev = otelSeverityToDataSetSeverity(log)
 
 	if timestamp := log.Timestamp().AsTime(); !timestamp.Equal(time.Unix(0, 0)) {
 		event.Ts = strconv.FormatInt(timestamp.UnixNano(), 10)
@@ -93,9 +159,6 @@ func buildEventFromLog(
 	if !observedTs.Equal(time.Unix(0, 0)) {
 		attrs["observed.timestamp"] = observedTs.String()
 	}
-	if sevText := log.SeverityText(); sevText != "" {
-		attrs["severity.text"] = sevText
-	}
 	if span := log.SpanID().String(); span != "" {
 		attrs["span_id"] = span
 	}

exporter/datasetexporter/logs_exporter_test.go

@@ -156,7 +156,7 @@ func TestBuildBodyMap(t *testing.T) {
 var testLEventRaw = &add_events.Event{
 	Thread: "TL",
 	Log:    "LL",
-	Sev:    9,
+	Sev:    3,
 	Ts:     "1581452773000000789",
 	Attrs: map[string]interface{}{
 		"attributes.app":           "server",
@@ -168,8 +168,6 @@ var testLEventRaw = &add_events.Event{
 		"flags":                    plog.LogRecordFlags(0),
 		"message":                  "This is a log message",
 		"scope.name":               "",
-		"severity.number":          plog.SeverityNumberInfo,
-		"severity.text":            "Info",
 		"span_id":                  "0102040800000000",
 		"trace_id":                 "08040201000000000000000000000000",
 	},
@@ -190,8 +188,6 @@ var testLEventReq = &add_events.Event{
 		"flags":                    float64(plog.LogRecordFlags(0)),
 		"message":                  "This is a log message",
 		"scope.name":               "",
-		"severity.number":          float64(plog.SeverityNumberInfo),
-		"severity.text":            "Info",
 		"span_id":                  "0102040800000000",
 		"trace_id":                 "08040201000000000000000000000000",
 		"bundle_key":               "d41d8cd98f00b204e9800998ecf8427e",
@@ -407,3 +403,155 @@ func TestConsumeLogsShouldSucceed(t *testing.T) {
 		addRequest,
 	)
 }
+
+func TestOtelSeverityToDataSetSeverityWithSeverityNumberNoSeverityText(t *testing.T) {
+	lr := testdata.GenerateLogsOneLogRecord()
+	ld := lr.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0)
+	ld.SetSeverityText("")
+
+	// trace
+	ld.SetSeverityNumber(1)
+	assert.Equal(t, 1, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(2)
+	assert.Equal(t, 1, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(3)
+	assert.Equal(t, 1, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(4)
+	assert.Equal(t, 1, otelSeverityToDataSetSeverity(ld))
+
+	// debug
+	ld.SetSeverityNumber(5)
+	assert.Equal(t, 2, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(6)
+	assert.Equal(t, 2, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(7)
+	assert.Equal(t, 2, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(8)
+	assert.Equal(t, 2, otelSeverityToDataSetSeverity(ld))
+
+	// info
+	ld.SetSeverityNumber(9)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(10)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(11)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(12)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	// warn
+	ld.SetSeverityNumber(13)
+	assert.Equal(t, 4, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(14)
+	assert.Equal(t, 4, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(15)
+	assert.Equal(t, 4, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(16)
+	assert.Equal(t, 4, otelSeverityToDataSetSeverity(ld))
+
+	// error
+	ld.SetSeverityNumber(17)
+	assert.Equal(t, 5, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(18)
+	assert.Equal(t, 5, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(19)
+	assert.Equal(t, 5, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(20)
+	assert.Equal(t, 5, otelSeverityToDataSetSeverity(ld))
+
+	// fatal
+	ld.SetSeverityNumber(21)
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(22)
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(22)
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(24)
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+	// Invalid values get mapped to info (3)
+	ld.SetSeverityNumber(0)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(-1)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(25)
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+}
+
+func TestOtelSeverityToDataSetSeverityWithSeverityTextNoSeverityNumber(t *testing.T) {
+	lr := testdata.GenerateLogsOneLogRecord()
+	ld := lr.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0)
+	ld.SetSeverityNumber(0)
+
+	// trace
+	ld.SetSeverityText("trace")
+	assert.Equal(t, 1, otelSeverityToDataSetSeverity(ld))
+
+	// debug
+	ld.SetSeverityText("debug")
+	assert.Equal(t, 2, otelSeverityToDataSetSeverity(ld))
+
+	// info
+	ld.SetSeverityText("info")
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityText("informational")
+	assert.Equal(t, 3, otelSeverityToDataSetSeverity(ld))
+
+	// warn
+	ld.SetSeverityText("warn")
+	assert.Equal(t, 4, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityText("warning")
+	assert.Equal(t, 4, otelSeverityToDataSetSeverity(ld))
+
+	// error
+	ld.SetSeverityText("error")
+	assert.Equal(t, 5, otelSeverityToDataSetSeverity(ld))
+
+	// fatal
+	ld.SetSeverityText("fatal")
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityText("critical")
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityText("emergency")
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+}
+
+func TestOtelSeverityToDataSetSeverityWithSeverityNumberAndSeverityTextSeverityNumberHasPriority(t *testing.T) {
+	// If provided, SeverityNumber has priority over SeverityText
+	lr := testdata.GenerateLogsOneLogRecord()
+	ld := lr.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0)
+
+	ld.SetSeverityNumber(3)
+	ld.SetSeverityText("debug")
+	assert.Equal(t, 1, otelSeverityToDataSetSeverity(ld))
+
+	ld.SetSeverityNumber(22)
+	ld.SetSeverityText("info")
+	assert.Equal(t, 6, otelSeverityToDataSetSeverity(ld))
+
+}