[pkg/stanza][receiver/windowseventlog] Improve EventDataType support in EventXML (Windows)

.chloggen/improve-event-data-support-for-windows-event-xml.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/stanza
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Improve parsing of Windows Event XML by handling anonymous `Data` elements.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [21491]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: This improves the contents of Windows log events for which the publisher manifest is unavailable. Previously, anonymous `Data` elements were ignored. This is a breaking change for users who were relying on the previous data format.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

pkg/stanza/operator/input/windows/testdata/xmlWithAnonymousEventDataEntries.xml

@@ -0,0 +1,23 @@
+<Event xmlns="http://schemas.microsoft.com/win/2004/08/events/event">
+  <System>
+    <Provider Name="VSS" /> 
+    <EventID Qualifiers="0">8194</EventID> 
+    <Version>0</Version> 
+    <Level>2</Level> 
+    <Task>0</Task> 
+    <Opcode>0</Opcode> 
+    <Keywords>0x80000000000000</Keywords> 
+    <TimeCreated SystemTime="2023-10-19T21:57:58.0685414Z" /> 
+    <EventRecordID>383972</EventRecordID> 
+    <Correlation /> 
+    <Execution ProcessID="0" ThreadID="0" /> 
+    <Channel>Application</Channel> 
+    <Computer>computer</Computer> 
+    <Security /> 
+  </System>
+  <EventData>
+    <Data>1st_value</Data> 
+    <Data>2nd_value</Data> 
+    <Binary>2D20</Binary> 
+  </EventData>
+</Event>

pkg/stanza/operator/input/windows/xml.go

@@ -13,24 +13,24 @@ import (
 
 // EventXML is the rendered xml of an event.
 type EventXML struct {
-	EventID          EventID          `xml:"System>EventID"`
-	Provider         Provider         `xml:"System>Provider"`
-	Computer         string           `xml:"System>Computer"`
-	Channel          string           `xml:"System>Channel"`
-	RecordID         uint64           `xml:"System>EventRecordID"`
-	TimeCreated      TimeCreated      `xml:"System>TimeCreated"`
-	Message          string           `xml:"RenderingInfo>Message"`
-	RenderedLevel    string           `xml:"RenderingInfo>Level"`
-	Level            string           `xml:"System>Level"`
-	RenderedTask     string           `xml:"RenderingInfo>Task"`
-	Task             string           `xml:"System>Task"`
-	RenderedOpcode   string           `xml:"RenderingInfo>Opcode"`
-	Opcode           string           `xml:"System>Opcode"`
-	RenderedKeywords []string         `xml:"RenderingInfo>Keywords>Keyword"`
-	Keywords         []string         `xml:"System>Keywords"`
-	Security         *Security        `xml:"System>Security"`
-	Execution        *Execution       `xml:"System>Execution"`
-	EventData        []EventDataEntry `xml:"EventData>Data"`
+	EventID          EventID     `xml:"System>EventID"`
+	Provider         Provider    `xml:"System>Provider"`
+	Computer         string      `xml:"System>Computer"`
+	Channel          string      `xml:"System>Channel"`
+	RecordID         uint64      `xml:"System>EventRecordID"`
+	TimeCreated      TimeCreated `xml:"System>TimeCreated"`
+	Message          string      `xml:"RenderingInfo>Message"`
+	RenderedLevel    string      `xml:"RenderingInfo>Level"`
+	Level            string      `xml:"System>Level"`
+	RenderedTask     string      `xml:"RenderingInfo>Task"`
+	Task             string      `xml:"System>Task"`
+	RenderedOpcode   string      `xml:"RenderingInfo>Opcode"`
+	Opcode           string      `xml:"System>Opcode"`
+	RenderedKeywords []string    `xml:"RenderingInfo>Keywords>Keyword"`
+	Keywords         []string    `xml:"System>Keywords"`
+	Security         *Security   `xml:"System>Security"`
+	Execution        *Execution  `xml:"System>Execution"`
+	EventData        EventData   `xml:"EventData"`
 }
 
 // parseTimestamp will parse the timestamp of the event.
@@ -148,19 +148,30 @@ func (e *EventXML) parseMessage() (string, map[string]interface{}) {
 	}
 }
 
-// parse event data entries into a map[string]interface
-// where the key is the Name attribute, and value is the element value
-// entries without Name are ignored
+// parse event data into a map[string]interface
 // see: https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-datafieldtype-complextype
-func parseEventData(entries []EventDataEntry) map[string]interface{} {
-	outputMap := make(map[string]interface{}, len(entries))
+func parseEventData(eventData EventData) map[string]interface{} {
+	outputMap := make(map[string]interface{}, 3)
+	if eventData.Name != "" {
+		outputMap["name"] = eventData.Name
+	}
+	if eventData.Binary != "" {
+		outputMap["binary"] = eventData.Binary
+	}
+
+	if len(eventData.Data) == 0 {
+		return outputMap
+	}
 
-	for _, entry := range entries {
-		if entry.Name != "" {
-			outputMap[entry.Name] = entry.Value
+	dataMaps := make([]interface{}, len(eventData.Data))
+	for i, data := range eventData.Data {
+		dataMaps[i] = map[string]interface{}{
+			data.Name: data.Value,
 		}
 	}
 
+	outputMap["data"] = dataMaps
+
 	return outputMap
 }
 
@@ -191,7 +202,16 @@ type Provider struct {
 	EventSourceName string `xml:"EventSourceName,attr"`
 }
 
-type EventDataEntry struct {
+type EventData struct {
+	// https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-eventdatatype-complextype
+	// ComplexData is not supported.
+	Name   string `xml:"Name,attr"`
+	Data   []Data `xml:"Data"`
+	Binary string `xml:"Binary"`
+}
+
+type Data struct {
+	// https://learn.microsoft.com/en-us/windows/win32/wes/eventschema-datafieldtype-complextype
 	Name  string `xml:"Name,attr"`
 	Value string `xml:",chardata"`
 }

pkg/stanza/operator/input/windows/xml_test.go

@@ -82,8 +82,8 @@ func TestParseBody(t *testing.T) {
 		Task:     "task",
 		Opcode:   "opcode",
 		Keywords: []string{"keyword"},
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
+		EventData: EventData{
+			Data: []Data{{Name: "1st_name", Value: "value"}, {Name: "2nd_name", Value: "another_value"}},
 		},
 		RenderedLevel:    "rendered_level",
 		RenderedTask:     "rendered_task",
@@ -110,7 +110,12 @@ func TestParseBody(t *testing.T) {
 		"task":        "rendered_task",
 		"opcode":      "rendered_opcode",
 		"keywords":    []string{"RenderedKeywords"},
-		"event_data":  map[string]interface{}{"name": "value", "another_name": "another_value"},
+		"event_data": map[string]interface{}{
+			"data": []interface{}{
+				map[string]interface{}{"1st_name": "value"},
+				map[string]interface{}{"2nd_name": "another_value"},
+			},
+		},
 	}
 
 	require.Equal(t, expected, xml.parseBody())
@@ -138,8 +143,8 @@ func TestParseBodySecurityExecution(t *testing.T) {
 		Task:     "task",
 		Opcode:   "opcode",
 		Keywords: []string{"keyword"},
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
+		EventData: EventData{
+			Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
 		},
 		Execution: &Execution{
 			ProcessID: 13,
@@ -180,7 +185,12 @@ func TestParseBodySecurityExecution(t *testing.T) {
 		"security": map[string]any{
 			"user_id": "my-user-id",
 		},
-		"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
+		"event_data": map[string]interface{}{
+			"data": []interface{}{
+				map[string]interface{}{"name": "value"},
+				map[string]interface{}{"another_name": "another_value"},
+			},
+		},
 	}
 
 	require.Equal(t, expected, xml.parseBody())
@@ -214,8 +224,8 @@ func TestParseBodyFullExecution(t *testing.T) {
 		Task:     "task",
 		Opcode:   "opcode",
 		Keywords: []string{"keyword"},
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
+		EventData: EventData{
+			Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
 		},
 		Execution: &Execution{
 			ProcessID:     13,
@@ -266,7 +276,12 @@ func TestParseBodyFullExecution(t *testing.T) {
 		"security": map[string]any{
 			"user_id": "my-user-id",
 		},
-		"event_data": map[string]interface{}{"name": "value", "another_name": "another_value"},
+		"event_data": map[string]interface{}{
+			"data": []interface{}{
+				map[string]interface{}{"name": "value"},
+				map[string]interface{}{"another_name": "another_value"},
+			},
+		},
 	}
 
 	require.Equal(t, expected, xml.parseBody())
@@ -294,8 +309,8 @@ func TestParseNoRendered(t *testing.T) {
 		Task:     "task",
 		Opcode:   "opcode",
 		Keywords: []string{"keyword"},
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
+		EventData: EventData{
+			Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
 		},
 	}
 
@@ -318,7 +333,12 @@ func TestParseNoRendered(t *testing.T) {
 		"task":        "task",
 		"opcode":      "opcode",
 		"keywords":    []string{"keyword"},
-		"event_data":  map[string]interface{}{"name": "value", "another_name": "another_value"},
+		"event_data": map[string]interface{}{
+			"data": []interface{}{
+				map[string]interface{}{"name": "value"},
+				map[string]interface{}{"another_name": "another_value"},
+			},
+		},
 	}
 
 	require.Equal(t, expected, xml.parseBody())
@@ -346,8 +366,8 @@ func TestParseBodySecurity(t *testing.T) {
 		Task:     "task",
 		Opcode:   "opcode",
 		Keywords: []string{"keyword"},
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"},
+		EventData: EventData{
+			Data: []Data{{Name: "name", Value: "value"}, {Name: "another_name", Value: "another_value"}},
 		},
 		RenderedLevel:    "rendered_level",
 		RenderedTask:     "rendered_task",
@@ -374,32 +394,48 @@ func TestParseBodySecurity(t *testing.T) {
 		"task":        "rendered_task",
 		"opcode":      "rendered_opcode",
 		"keywords":    []string{"RenderedKeywords"},
-		"event_data":  map[string]interface{}{"name": "value", "another_name": "another_value"},
+		"event_data": map[string]interface{}{
+			"data": []interface{}{
+				map[string]interface{}{"name": "value"},
+				map[string]interface{}{"another_name": "another_value"},
+			},
+		},
 	}
 
 	require.Equal(t, expected, xml.parseBody())
 }
 
 func TestParseEventData(t *testing.T) {
 	xmlMap := EventXML{
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"},
+		EventData: EventData{
+			Name:   "EVENT_DATA",
+			Data:   []Data{{Name: "name", Value: "value"}},
+			Binary: "2D20",
 		},
 	}
 
 	parsed := xmlMap.parseBody()
-	expectedMap := map[string]interface{}{"name": "value"}
+	expectedMap := map[string]interface{}{
+		"name": "EVENT_DATA",
+		"data": []interface{}{
+			map[string]interface{}{"name": "value"},
+		},
+		"binary": "2D20",
+	}
 	require.Equal(t, expectedMap, parsed["event_data"])
 
 	xmlMixed := EventXML{
-		EventData: []EventDataEntry{
-			{Name: "name", Value: "value"},
-			{Value: "noname"},
+		EventData: EventData{
+			Data: []Data{{Name: "name", Value: "value"}, {Value: "no_name"}},
 		},
 	}
 
 	parsed = xmlMixed.parseBody()
-	expectedSlice := map[string]interface{}{"name": "value"}
+	expectedSlice := map[string]interface{}{
+		"data": []interface{}{
+			map[string]interface{}{"name": "value"},
+			map[string]interface{}{"": "no_name"}},
+	}
 	require.Equal(t, expectedSlice, parsed["event_data"])
 }
 
@@ -428,16 +464,54 @@ func TestUnmarshalWithEventData(t *testing.T) {
 		TimeCreated: TimeCreated{
 			SystemTime: "2022-04-22T10:20:52.3778625Z",
 		},
+		Computer:  "computer",
+		Channel:   "Application",
+		RecordID:  23401,
+		Level:     "4",
+		Message:   "",
+		Task:      "0",
+		Opcode:    "0",
+		Execution: &Execution{},
+		Security:  &Security{},
+		EventData: EventData{
+			Data: []Data{
+				{Name: "Time", Value: "2022-04-28T19:48:52Z"},
+				{Name: "Source", Value: "RulesEngine"}},
+		},
+		Keywords: []string{"0x80000000000000"},
+	}
+
+	require.Equal(t, xml, event)
+}
+
+func TestUnmarshalWithAnonymousEventDataEntries(t *testing.T) {
+	data, err := os.ReadFile(filepath.Join("testdata", "xmlWithAnonymousEventDataEntries.xml"))
+	require.NoError(t, err)
+
+	event, err := unmarshalEventXML(data)
+	require.NoError(t, err)
+
+	xml := EventXML{
+		EventID: EventID{
+			ID:         8194,
+			Qualifiers: 0,
+		},
+		Provider: Provider{
+			Name: "VSS",
+		},
+		TimeCreated: TimeCreated{
+			SystemTime: "2023-10-19T21:57:58.0685414Z",
+		},
 		Computer: "computer",
 		Channel:  "Application",
-		RecordID: 23401,
-		Level:    "4",
+		RecordID: 383972,
+		Level:    "2",
 		Message:  "",
 		Task:     "0",
 		Opcode:   "0",
-		EventData: []EventDataEntry{
-			{Name: "Time", Value: "2022-04-28T19:48:52Z"},
-			{Name: "Source", Value: "RulesEngine"},
+		EventData: EventData{
+			Data:   []Data{{Name: "", Value: "1st_value"}, {Name: "", Value: "2nd_value"}},
+			Binary: "2D20",
 		},
 		Keywords:  []string{"0x80000000000000"},
 		Security:  &Security{},

receiver/windowseventlogreceiver/receiver_windows_test.go

@@ -117,7 +117,9 @@ func TestReadWindowsEventLogger(t *testing.T) {
 	eventData := body["event_data"]
 	eventDataMap, ok := eventData.(map[string]interface{})
 	require.True(t, ok)
-	require.Equal(t, map[string]interface{}{}, eventDataMap)
+	require.Equal(t, map[string]interface{}{
+		"data": []interface{}{map[string]interface{}{"": "Test log"}},
+	}, eventDataMap)
 
 	eventID := body["event_id"]
 	require.NotNil(t, eventID)