Fix logs display when _source is log message field (#565)

grafana · Feb 5, 2025 · 28f2a50 · 28f2a50
1 parent 4b18010
commit 28f2a50
Show file tree

Hide file tree

Showing 5 changed files with 144 additions and 2,924 deletions.
diff --git a/cspell.config.json b/cspell.config.json
@@ -8,7 +8,8 @@
     "go.sum",
     "go.mod",
     ".gitignore",
-    ".config/**"
+    ".config/**",
+    "pkg/opensearch/snapshot_tests/testdata/**"
   ],
   "words": [
     "aggs",

diff --git a/pkg/opensearch/client/client.go b/pkg/opensearch/client/client.go
@@ -119,7 +119,8 @@ func NewClient(ctx context.Context, ds *backend.DataSourceInstanceSettings, http
 	}
 
 	logLevelField := jsonData.Get("logLevelField").MustString()
-	logMessageField := jsonData.Get("logMessageField").MustString()
+	// logMessageField defaults to "_source"
+	logMessageField := jsonData.Get("logMessageField").MustString("_source")
 
 	db, err := jsonData.Get("database").String()
 	if err != nil {

diff --git a/pkg/opensearch/response_parser.go b/pkg/opensearch/response_parser.go
@@ -482,15 +482,23 @@ func processLogsResponse(res *client.SearchResponse, configuredFields client.Con
 
 	for hitIdx, hit := range res.Hits.Hits {
 		var flattened map[string]interface{}
+		var sourceString string
 		if hit["_source"] != nil {
 			flattened = flatten(hit["_source"].(map[string]interface{}), maxFlattenDepth)
+			sourceMarshalled, err := json.Marshal(flattened)
+			if err != nil {
+				errResp := errorsource.Response(errorsource.PluginError(err, false))
+				return errResp
+			}
+			sourceString = string(sourceMarshalled)
 		}
 
 		doc := map[string]interface{}{
-			"_id":     hit["_id"],
-			"_type":   hit["_type"],
-			"_index":  hit["_index"],
-			"_source": flattened,
+			"_id":    hit["_id"],
+			"_type":  hit["_type"],
+			"_index": hit["_index"],
+			// In case of logs query we want to have the raw source as a string field so it can be visualized in logs panel
+			"_source": sourceString,
 		}
 
 		for k, v := range flattened {

diff --git a/pkg/opensearch/response_parser_test.go b/pkg/opensearch/response_parser_test.go
@@ -1488,9 +1488,9 @@ func TestProcessLogsResponse_creates_correct_data_frame_fields(t *testing.T) {
 				utils.Pointer("mock-index"),
 			}).SetConfig(&data.FieldConfig{Filterable: utils.Pointer(true)}),
 		data.NewField("_source", nil,
-			[]*json.RawMessage{
-				utils.Pointer(json.RawMessage(`{"fields.lvl":"debug","host":"djisaodjsoad","level":"debug","line":"hello, i am a message","number":1,"testtime":"06/24/2019"}`)),
-				utils.Pointer(json.RawMessage(`{"fields.lvl":"info","host":"dsalkdakdop","level":"error","line":"hello, i am also message","number":2,"testtime":"06/24/2019"}`)),
+			[]*string{
+				utils.Pointer(`{"fields.lvl":"debug","host":"djisaodjsoad","level":"debug","line":"hello, i am a message","number":1,"testtime":"06/24/2019"}`),
+				utils.Pointer(`{"fields.lvl":"info","host":"dsalkdakdop","level":"error","line":"hello, i am also message","number":2,"testtime":"06/24/2019"}`),
 			}).SetConfig(&data.FieldConfig{Filterable: utils.Pointer(true)}),
 		data.NewField("_type", nil,
 			[]*string{
@@ -1697,9 +1697,9 @@ func TestProcessLogsResponse_log_query_with_nested_fields(t *testing.T) {
 				utils.Pointer("logs-2023.02.08"),
 			}).SetConfig(&data.FieldConfig{Filterable: utils.Pointer(true)}),
 		data.NewField("_source", nil,
-			[]*json.RawMessage{
-				utils.Pointer(json.RawMessage(`{"@timestamp":"2023-02-08T15:10:55.830Z","counter":"109","float":58.253758485091,"label":"val1","line":"log text  [479231733]","location":"17.089705232090438, 41.62861966340297","lvl":"info","nested.field.double_nested":true,"shapes":[{"type":"triangle"},{"type":"square"}],"xyz":null}`)),
-				utils.Pointer(json.RawMessage(`{"@timestamp":"2023-02-08T15:10:54.835Z","counter":"108","float":54.5977098233944,"label":"val1","line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]","location":"19.766305918490463, 40.42639175509792","lvl":"info","nested.field.double_nested":false,"shapes":[{"type":"triangle"},{"type":"square"}],"xyz":"def"}`)),
+			[]*string{
+				utils.Pointer(`{"@timestamp":"2023-02-08T15:10:55.830Z","counter":"109","float":58.253758485091,"label":"val1","line":"log text  [479231733]","location":"17.089705232090438, 41.62861966340297","lvl":"info","nested.field.double_nested":true,"shapes":[{"type":"triangle"},{"type":"square"}],"xyz":null}`),
+				utils.Pointer(`{"@timestamp":"2023-02-08T15:10:54.835Z","counter":"108","float":54.5977098233944,"label":"val1","line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]","location":"19.766305918490463, 40.42639175509792","lvl":"info","nested.field.double_nested":false,"shapes":[{"type":"triangle"},{"type":"square"}],"xyz":"def"}`),
 			}).SetConfig(&data.FieldConfig{Filterable: utils.Pointer(true)}),
 		data.NewField("_type", nil,
 			[]*json.RawMessage{