Allow transform_generic to keep existing fields

README.md

@@ -249,6 +249,7 @@ The generic transform module maps the input json keys into another set of keys.
 This allows to perform subsequent operations using a uniform set of keys.
 In some use cases, only a subset of the provided fields are required.
 Using the generic transform, we may specify those particular fields that interest us.
+To include keys that are not specified in the `rules`, add the line `maintain: true`.
 
 For example, suppose we have a flow log with the following syntax:
 ```
@@ -295,7 +296,7 @@ pipeline:
     follows: transform1
 parameters:
   - name: transform1
-    transform
+    transform:
       type: generic
       generic:
         rules:
@@ -304,7 +305,7 @@ parameters:
           - input: SrcAddr
             output: srcAddr
   - name: transform2
-    transform
+    transform:
       type: generic
       generic:
         rules:
@@ -321,6 +322,19 @@ Before the first transform suppose we have the keys `DstAddr` and `SrcAddr`.
 After the first transform, we have the keys `dstAddr` and `srcAddr`.
 After the second transform, we have the keys `dstAddr`, `dstIP`, `srcAddr`, and `srcIP`.
 
+To maintain all the old key/values and change only the key `DstAddr` to `dstAddr`, use the following:
+```
+parameters:
+  - name: transform1
+    transform:
+      type: generic
+      generic:
+        maintain: true
+        rules:
+          - input: DstAddr
+            output: dstAddr
+```
+
 ### Transform Filter
 
 The filter transform module allows setting rules to remove complete entries from

docs/api.md

@@ -71,6 +71,7 @@ Following is the supported API format for generic transformations:
 
 <pre>
  generic:
+         maintain: maintain old fields that are not specified (default: false)
          rules: list of transform rules, each includes:
                  input: entry input field
                  output: entry output field

pkg/api/transform_generic.go

@@ -18,7 +18,8 @@
 package api
 
 type TransformGeneric struct {
-	Rules []GenericTransformRule `yaml:"rules" doc:"list of transform rules, each includes:"`
+	Maintain bool                   `yaml:"maintain" doc:"maintain old fields that are not specified (default: false)"`
+	Rules    []GenericTransformRule `yaml:"rules" doc:"list of transform rules, each includes:"`
 }
 
 type GenericTransformRule struct {

pkg/pipeline/transform/transform_generic.go

@@ -24,7 +24,9 @@ import (
 )
 
 type Generic struct {
-	Rules []api.GenericTransformRule
+	maintain  bool
+	rules     []api.GenericTransformRule
+	mapOfKeys map[string]bool
 }
 
 // Transform transforms a flow to a new set of keys
@@ -33,21 +35,44 @@ func (g *Generic) Transform(input []config.GenericMap) []config.GenericMap {
 	output := make([]config.GenericMap, 0)
 	for _, entry := range input {
 		outputEntry := make(config.GenericMap)
-		for _, transformRule := range g.Rules {
+		for _, transformRule := range g.rules {
 			log.Debugf("transformRule = %v", transformRule)
 			outputEntry[transformRule.Output] = entry[transformRule.Input]
 		}
+		if g.maintain {
+			g.MaintainEntrichment(outputEntry, entry)
+		}
 		log.Debugf("Transform.GenericMap = %v", outputEntry)
 		output = append(output, outputEntry)
 	}
 	return output
 }
 
+func (g *Generic) MaintainEntrichment(outputEntry, inputEntry config.GenericMap) {
+	log.Debugf("entering MaintainEntrichment")
+	for key, value := range inputEntry {
+		// if key appears in list of Rules (mapOfKeys), it was already handled
+		if g.mapOfKeys[key] {
+			continue
+		}
+		outputEntry[key] = value
+	}
+}
+
 // NewTransformGeneric create a new transform
 func NewTransformGeneric(params config.StageParam) (Transformer, error) {
 	log.Debugf("entering NewTransformGeneric")
+	log.Debugf("params.Transform.Generic = %v", params.Transform.Generic)
+	mapOfKeys := make(map[string]bool)
+	rules := params.Transform.Generic.Rules
+	for _, rule := range rules {
+		mapOfKeys[rule.Input] = true
+	}
 	transformGeneric := &Generic{
-		Rules: params.Transform.Generic.Rules,
+		maintain:  params.Transform.Generic.Maintain,
+		rules:     rules,
+		mapOfKeys: mapOfKeys,
 	}
+	log.Debugf("transformGeneric = %v", transformGeneric)
 	return transformGeneric, nil
 }

pkg/pipeline/transform/transform_generic_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const testConfigTransformGeneric = `---
+const testConfigTransformGenericDefault = `---
 log-level: debug
 pipeline:
   - name: transform1
@@ -48,8 +48,57 @@ parameters:
         - input: srcIP
           output: srcIP
 `
+const testConfigTransformGenericMaintainFalse = `---
+log-level: debug
+pipeline:
+  - name: transform1
+parameters:
+  - name: transform1
+    transform:
+      type: generic
+      generic:
+        maintain: false
+        rules:
+        - input: srcIP
+          output: SrcAddr
+        - input: dstIP
+          output: DstAddr
+        - input: dstPort
+          output: DstPort
+        - input: srcPort
+          output: SrcPort
+        - input: protocol
+          output: Protocol
+        - input: srcIP
+          output: srcIP
+`
 
-func getGenericExpectedOutput() config.GenericMap {
+const testConfigTransformGenericMaintainTrue = `---
+log-level: debug
+pipeline:
+  - name: transform1
+parameters:
+  - name: transform1
+    transform:
+      type: generic
+      generic:
+        maintain: true
+        rules:
+        - input: srcIP
+          output: SrcAddr
+        - input: dstIP
+          output: DstAddr
+        - input: dstPort
+          output: DstPort
+        - input: srcPort
+          output: SrcPort
+        - input: protocol
+          output: Protocol
+        - input: srcIP
+          output: srcIP
+`
+
+func getGenericExpectedOutputShort() config.GenericMap {
 	return config.GenericMap{
 		"SrcAddr":  "10.0.0.1",
 		"srcIP":    "10.0.0.1",
@@ -60,14 +109,53 @@ func getGenericExpectedOutput() config.GenericMap {
 	}
 }
 
-func TestNewTransformGeneric(t *testing.T) {
-	newTransform := InitNewTransformGeneric(t, testConfigTransformGeneric)
+func getGenericExpectedOutputLong() config.GenericMap {
+	return config.GenericMap{
+		"SrcAddr":      "10.0.0.1",
+		"srcIP":        "10.0.0.1",
+		"SrcPort":      11777,
+		"Protocol":     "tcp",
+		"DstAddr":      "20.0.0.2",
+		"DstPort":      22,
+		"8888IP":       "8.8.8.8",
+		"emptyIP":      "",
+		"level":        "error",
+		"protocol_num": 6,
+		"value":        7.0,
+		"message":      "test message",
+	}
+}
+
+func TestNewTransformGenericDefault(t *testing.T) {
+	newTransform := InitNewTransformGeneric(t, testConfigTransformGenericDefault)
+	transformGeneric := newTransform.(*Generic)
+	require.Len(t, transformGeneric.rules, 6)
+
+	input := test.GetIngestMockEntry(false)
+	output := transformGeneric.Transform([]config.GenericMap{input})
+	expectedOutput := getGenericExpectedOutputShort()
+	require.Equal(t, expectedOutput, output[0])
+}
+
+func TestNewTransformGenericMaintainFalse(t *testing.T) {
+	newTransform := InitNewTransformGeneric(t, testConfigTransformGenericMaintainFalse)
+	transformGeneric := newTransform.(*Generic)
+	require.Len(t, transformGeneric.rules, 6)
+
+	input := test.GetIngestMockEntry(false)
+	output := transformGeneric.Transform([]config.GenericMap{input})
+	expectedOutput := getGenericExpectedOutputShort()
+	require.Equal(t, expectedOutput, output[0])
+}
+
+func TestNewTransformGenericMaintainTrue(t *testing.T) {
+	newTransform := InitNewTransformGeneric(t, testConfigTransformGenericMaintainTrue)
 	transformGeneric := newTransform.(*Generic)
-	require.Len(t, transformGeneric.Rules, 6)
+	require.Len(t, transformGeneric.rules, 6)
 
 	input := test.GetIngestMockEntry(false)
 	output := transformGeneric.Transform([]config.GenericMap{input})
-	expectedOutput := getGenericExpectedOutput()
+	expectedOutput := getGenericExpectedOutputLong()
 	require.Equal(t, expectedOutput, output[0])
 }