[processor/transform] Add new delete functions (#11824)

* Add new delete functions

* Updated changelog

processor/transformprocessor/README.md

@@ -34,6 +34,10 @@ Supported functions:
 e.g., `set(attributes["http.path"], "/foo")`, `set(name, attributes["http.route"])`, `set(trace_state["svc"], "example")`, `set(attributes["source"], trace_state["source"])`. If `value` resolves to `nil`, e.g.
 it references an unset map value, there will be no action.
 
+- `delete_key(target, key)` - `target` is a path expression to a map type field. `key` is a string that is a key in the map.  The key will be deleted from the map.  e.g., `delete_key(attributes, "http.request.header.authorization")`
+
+- `delete_matching_keys(target, pattern)` - `target` is a path expression to a map type field. `pattern` is a regex string.  All keys that match the pattern will be deleted from the map.  e.g., `delete_matching_keys(attributes, ".*\.header\.authorization")`
+
 - `keep_keys(target, string...)` - `target` is a path expression to a map type field. The map will be mutated to only contain
 the fields specified by the list of strings. e.g., `keep_keys(attributes, "http.method")`, `keep_keys(attributes, "http.method", "http.route")`
 

processor/transformprocessor/internal/common/func_delete_key.go

@@ -0,0 +1,31 @@
+// Copyright  The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common"
+
+import "go.opentelemetry.io/collector/pdata/pcommon"
+
+func deleteKey(target Getter, key string) (ExprFunc, error) {
+	return func(ctx TransformContext) interface{} {
+		val := target.Get(ctx)
+		if val == nil {
+			return nil
+		}
+
+		if attrs, ok := val.(pcommon.Map); ok {
+			attrs.Remove(key)
+		}
+		return nil
+	}, nil
+}

processor/transformprocessor/internal/common/func_delete_key_test.go

@@ -0,0 +1,137 @@
+// Copyright  The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common/testhelper"
+)
+
+func Test_deleteKey(t *testing.T) {
+	input := pcommon.NewMap()
+	input.InsertString("test", "hello world")
+	input.InsertInt("test2", 3)
+	input.InsertBool("test3", true)
+
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			return ctx.GetItem()
+		},
+	}
+
+	tests := []struct {
+		name   string
+		target Getter
+		key    string
+		want   func(pcommon.Map)
+	}{
+		{
+			name:   "delete test",
+			target: target,
+			key:    "test",
+			want: func(expectedMap pcommon.Map) {
+				expectedMap.Clear()
+				expectedMap.InsertBool("test3", true)
+				expectedMap.InsertInt("test2", 3)
+			},
+		},
+		{
+			name:   "delete test2",
+			target: target,
+			key:    "test2",
+			want: func(expectedMap pcommon.Map) {
+				expectedMap.Clear()
+				expectedMap.InsertString("test", "hello world")
+				expectedMap.InsertBool("test3", true)
+			},
+		},
+		{
+			name:   "delete nothing",
+			target: target,
+			key:    "not a valid key",
+			want: func(expectedMap pcommon.Map) {
+				expectedMap.Clear()
+				expectedMap.InsertString("test", "hello world")
+				expectedMap.InsertInt("test2", 3)
+				expectedMap.InsertBool("test3", true)
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scenarioMap := pcommon.NewMap()
+			input.CopyTo(scenarioMap)
+
+			ctx := testhelper.TestTransformContext{
+				Item: scenarioMap,
+			}
+
+			exprFunc, _ := deleteKey(tt.target, tt.key)
+			exprFunc(ctx)
+
+			expected := pcommon.NewMap()
+			tt.want(expected)
+
+			assert.Equal(t, expected, scenarioMap)
+		})
+	}
+}
+
+func Test_deleteKey_bad_input(t *testing.T) {
+	input := pcommon.NewValueString("not a map")
+	ctx := testhelper.TestTransformContext{
+		Item: input,
+	}
+
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			return ctx.GetItem()
+		},
+		setter: func(ctx TransformContext, val interface{}) {
+			t.Errorf("nothing should be set in this scenario")
+		},
+	}
+
+	key := "anything"
+
+	exprFunc, _ := deleteKey(target, key)
+	exprFunc(ctx)
+
+	assert.Equal(t, pcommon.NewValueString("not a map"), input)
+}
+
+func Test_deleteKey_get_nil(t *testing.T) {
+	ctx := testhelper.TestTransformContext{
+		Item: nil,
+	}
+
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			return ctx.GetItem()
+		},
+		setter: func(ctx TransformContext, val interface{}) {
+			t.Errorf("nothing should be set in this scenario")
+		},
+	}
+
+	key := "anything"
+
+	exprFunc, _ := deleteKey(target, key)
+	exprFunc(ctx)
+}

processor/transformprocessor/internal/common/func_delete_matching_keys.go

@@ -0,0 +1,42 @@
+// Copyright  The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common"
+
+import (
+	"fmt"
+	"regexp"
+
+	"go.opentelemetry.io/collector/pdata/pcommon"
+)
+
+func deleteMatchingKeys(target Getter, pattern string) (ExprFunc, error) {
+	compiledPattern, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, fmt.Errorf("the regex pattern supplied to delete_matching_keys is not a valid pattern: %w", err)
+	}
+	return func(ctx TransformContext) interface{} {
+		val := target.Get(ctx)
+		if val == nil {
+			return nil
+		}
+
+		if attrs, ok := val.(pcommon.Map); ok {
+			attrs.RemoveIf(func(key string, _ pcommon.Value) bool {
+				return compiledPattern.MatchString(key)
+			})
+		}
+		return nil
+	}, nil
+}

processor/transformprocessor/internal/common/func_delete_matching_keys_test.go

@@ -0,0 +1,140 @@
+// Copyright  The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common/testhelper"
+)
+
+func Test_deleteMatchingKeys(t *testing.T) {
+	input := pcommon.NewMap()
+	input.InsertString("test", "hello world")
+	input.InsertInt("test2", 3)
+	input.InsertBool("test3", true)
+
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			return ctx.GetItem()
+		},
+	}
+
+	tests := []struct {
+		name    string
+		target  Getter
+		pattern string
+		want    func(pcommon.Map)
+	}{
+		{
+			name:    "delete everything",
+			target:  target,
+			pattern: "test.*",
+			want: func(expectedMap pcommon.Map) {
+				expectedMap.Clear()
+				expectedMap.EnsureCapacity(3)
+			},
+		},
+		{
+			name:    "delete attributes that end in a number",
+			target:  target,
+			pattern: "\\d$",
+			want: func(expectedMap pcommon.Map) {
+				expectedMap.Clear()
+				expectedMap.InsertString("test", "hello world")
+			},
+		},
+		{
+			name:    "delete nothing",
+			target:  target,
+			pattern: "not a matching pattern",
+			want: func(expectedMap pcommon.Map) {
+				expectedMap.Clear()
+				expectedMap.InsertString("test", "hello world")
+				expectedMap.InsertInt("test2", 3)
+				expectedMap.InsertBool("test3", true)
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scenarioMap := pcommon.NewMap()
+			input.CopyTo(scenarioMap)
+
+			ctx := testhelper.TestTransformContext{
+				Item: scenarioMap,
+			}
+
+			exprFunc, _ := deleteMatchingKeys(tt.target, tt.pattern)
+			exprFunc(ctx)
+
+			expected := pcommon.NewMap()
+			tt.want(expected)
+
+			assert.Equal(t, expected, scenarioMap)
+		})
+	}
+}
+
+func Test_deleteMatchingKeys_bad_input(t *testing.T) {
+	input := pcommon.NewValueInt(1)
+	ctx := testhelper.TestTransformContext{
+		Item: input,
+	}
+
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			return ctx.GetItem()
+		},
+	}
+
+	exprFunc, err := deleteMatchingKeys(target, "anything")
+	assert.Nil(t, err)
+	exprFunc(ctx)
+
+	assert.Equal(t, pcommon.NewValueInt(1), input)
+}
+
+func Test_deleteMatchingKeys_get_nil(t *testing.T) {
+	ctx := testhelper.TestTransformContext{
+		Item: nil,
+	}
+
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			return ctx.GetItem()
+		},
+	}
+
+	exprFunc, _ := deleteMatchingKeys(target, "anything")
+	exprFunc(ctx)
+}
+
+func Test_deleteMatchingKeys_invalid_pattern(t *testing.T) {
+	target := &testGetSetter{
+		getter: func(ctx TransformContext) interface{} {
+			t.Errorf("nothing should be received in this scenario")
+			return nil
+		},
+	}
+
+	invalidRegexPattern := "*"
+	exprFunc, err := deleteMatchingKeys(target, invalidRegexPattern)
+	assert.Nil(t, exprFunc)
+	assert.Contains(t, err.Error(), "error parsing regexp:")
+}

processor/transformprocessor/internal/common/functions.go

@@ -31,6 +31,8 @@ var registry = map[string]interface{}{
 	"replace_all_matches":  replaceAllMatches,
 	"replace_pattern":      replacePattern,
 	"replace_all_patterns": replaceAllPatterns,
+	"delete_key":           deleteKey,
+	"delete_matching_keys": deleteMatchingKeys,
 }
 
 type PathExpressionParser func(*Path) (GetSetter, error)

processor/transformprocessor/internal/logs/processor_test.go

@@ -109,6 +109,21 @@ func TestProcess(t *testing.T) {
 				td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("test", "pass")
 			},
 		},
+		{
+			query: `delete_key(attributes, "http.url") where body == "operationA"`,
+			want: func(td plog.Logs) {
+				td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear()
+				td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.method", "get")
+				td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.path", "/health")
+			},
+		},
+		{
+			query: `delete_matching_keys(attributes, "http.*t.*") where body == "operationA"`,
+			want: func(td plog.Logs) {
+				td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear()
+				td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.url", "http://localhost/health")
+			},
+		},
 	}
 
 	for _, tt := range tests {