From bded0dd400d3aa6a55227e8aba02742668db6629 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:14:13 -0600 Subject: [PATCH 1/2] Add new delete functions --- processor/transformprocessor/README.md | 4 + .../internal/common/func_delete_key.go | 31 ++++ .../internal/common/func_delete_key_test.go | 137 +++++++++++++++++ .../common/func_delete_matching_keys.go | 42 ++++++ .../common/func_delete_matching_keys_test.go | 140 ++++++++++++++++++ .../internal/common/functions.go | 2 + .../internal/logs/processor_test.go | 15 ++ .../internal/metrics/processor_test.go | 21 +++ .../internal/traces/processor_test.go | 15 ++ 9 files changed, 407 insertions(+) create mode 100644 processor/transformprocessor/internal/common/func_delete_key.go create mode 100644 processor/transformprocessor/internal/common/func_delete_key_test.go create mode 100644 processor/transformprocessor/internal/common/func_delete_matching_keys.go create mode 100644 processor/transformprocessor/internal/common/func_delete_matching_keys_test.go diff --git a/processor/transformprocessor/README.md b/processor/transformprocessor/README.md index cb4482f0ffd2..fb6edfb7458f 100644 --- a/processor/transformprocessor/README.md +++ b/processor/transformprocessor/README.md @@ -33,6 +33,10 @@ Supported functions: e.g., `set(attributes["http.path"], "/foo")`, `set(name, attributes["http.route"])`, `set(trace_state["svc"], "example")`, `set(attributes["source"], trace_state["source"])`. If `value` resolves to `nil`, e.g. it references an unset map value, there will be no action. +- `delete_key(target, key)` - `target` is a path expression to a map type field. `key` is a string that is a key in the map. The key will be deleted from the map. e.g., `delete_key(attributes, "http.request.header.authorization")` + +- `delete_matching_keys(target, pattern)` - `target` is a path expression to a map type field. `pattern` is a regex string. All keys that match the pattern will be deleted from the map. e.g., `delete_matching_keys(attributes, ".*\.header\.authorization")` + - `keep_keys(target, string...)` - `target` is a path expression to a map type field. The map will be mutated to only contain the fields specified by the list of strings. e.g., `keep_keys(attributes, "http.method")`, `keep_keys(attributes, "http.method", "http.route")` diff --git a/processor/transformprocessor/internal/common/func_delete_key.go b/processor/transformprocessor/internal/common/func_delete_key.go new file mode 100644 index 000000000000..8e26a44ed763 --- /dev/null +++ b/processor/transformprocessor/internal/common/func_delete_key.go @@ -0,0 +1,31 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common" + +import "go.opentelemetry.io/collector/pdata/pcommon" + +func deleteKey(target Getter, key string) (ExprFunc, error) { + return func(ctx TransformContext) interface{} { + val := target.Get(ctx) + if val == nil { + return nil + } + + if attrs, ok := val.(pcommon.Map); ok { + attrs.Remove(key) + } + return nil + }, nil +} diff --git a/processor/transformprocessor/internal/common/func_delete_key_test.go b/processor/transformprocessor/internal/common/func_delete_key_test.go new file mode 100644 index 000000000000..55018c77e95b --- /dev/null +++ b/processor/transformprocessor/internal/common/func_delete_key_test.go @@ -0,0 +1,137 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common/testhelper" +) + +func Test_deleteKey(t *testing.T) { + input := pcommon.NewMap() + input.InsertString("test", "hello world") + input.InsertInt("test2", 3) + input.InsertBool("test3", true) + + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + return ctx.GetItem() + }, + } + + tests := []struct { + name string + target Getter + key string + want func(pcommon.Map) + }{ + { + name: "delete test", + target: target, + key: "test", + want: func(expectedMap pcommon.Map) { + expectedMap.Clear() + expectedMap.InsertBool("test3", true) + expectedMap.InsertInt("test2", 3) + }, + }, + { + name: "delete test2", + target: target, + key: "test2", + want: func(expectedMap pcommon.Map) { + expectedMap.Clear() + expectedMap.InsertString("test", "hello world") + expectedMap.InsertBool("test3", true) + }, + }, + { + name: "delete nothing", + target: target, + key: "not a valid key", + want: func(expectedMap pcommon.Map) { + expectedMap.Clear() + expectedMap.InsertString("test", "hello world") + expectedMap.InsertInt("test2", 3) + expectedMap.InsertBool("test3", true) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scenarioMap := pcommon.NewMap() + input.CopyTo(scenarioMap) + + ctx := testhelper.TestTransformContext{ + Item: scenarioMap, + } + + exprFunc, _ := deleteKey(tt.target, tt.key) + exprFunc(ctx) + + expected := pcommon.NewMap() + tt.want(expected) + + assert.Equal(t, expected, scenarioMap) + }) + } +} + +func Test_deleteKey_bad_input(t *testing.T) { + input := pcommon.NewValueString("not a map") + ctx := testhelper.TestTransformContext{ + Item: input, + } + + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + return ctx.GetItem() + }, + setter: func(ctx TransformContext, val interface{}) { + t.Errorf("nothing should be set in this scenario") + }, + } + + key := "anything" + + exprFunc, _ := deleteKey(target, key) + exprFunc(ctx) + + assert.Equal(t, pcommon.NewValueString("not a map"), input) +} + +func Test_deleteKey_get_nil(t *testing.T) { + ctx := testhelper.TestTransformContext{ + Item: nil, + } + + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + return ctx.GetItem() + }, + setter: func(ctx TransformContext, val interface{}) { + t.Errorf("nothing should be set in this scenario") + }, + } + + key := "anything" + + exprFunc, _ := deleteKey(target, key) + exprFunc(ctx) +} diff --git a/processor/transformprocessor/internal/common/func_delete_matching_keys.go b/processor/transformprocessor/internal/common/func_delete_matching_keys.go new file mode 100644 index 000000000000..6893a42411bf --- /dev/null +++ b/processor/transformprocessor/internal/common/func_delete_matching_keys.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common" + +import ( + "fmt" + "regexp" + + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func deleteMatchingKeys(target Getter, pattern string) (ExprFunc, error) { + compiledPattern, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("the regex pattern supplied to delete_matching_keys is not a valid pattern: %w", err) + } + return func(ctx TransformContext) interface{} { + val := target.Get(ctx) + if val == nil { + return nil + } + + if attrs, ok := val.(pcommon.Map); ok { + attrs.RemoveIf(func(key string, _ pcommon.Value) bool { + return compiledPattern.MatchString(key) + }) + } + return nil + }, nil +} diff --git a/processor/transformprocessor/internal/common/func_delete_matching_keys_test.go b/processor/transformprocessor/internal/common/func_delete_matching_keys_test.go new file mode 100644 index 000000000000..2894a05125dc --- /dev/null +++ b/processor/transformprocessor/internal/common/func_delete_matching_keys_test.go @@ -0,0 +1,140 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/common/testhelper" +) + +func Test_deleteMatchingKeys(t *testing.T) { + input := pcommon.NewMap() + input.InsertString("test", "hello world") + input.InsertInt("test2", 3) + input.InsertBool("test3", true) + + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + return ctx.GetItem() + }, + } + + tests := []struct { + name string + target Getter + pattern string + want func(pcommon.Map) + }{ + { + name: "delete everything", + target: target, + pattern: "test.*", + want: func(expectedMap pcommon.Map) { + expectedMap.Clear() + expectedMap.EnsureCapacity(3) + }, + }, + { + name: "delete attributes that end in a number", + target: target, + pattern: "\\d$", + want: func(expectedMap pcommon.Map) { + expectedMap.Clear() + expectedMap.InsertString("test", "hello world") + }, + }, + { + name: "delete nothing", + target: target, + pattern: "not a matching pattern", + want: func(expectedMap pcommon.Map) { + expectedMap.Clear() + expectedMap.InsertString("test", "hello world") + expectedMap.InsertInt("test2", 3) + expectedMap.InsertBool("test3", true) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scenarioMap := pcommon.NewMap() + input.CopyTo(scenarioMap) + + ctx := testhelper.TestTransformContext{ + Item: scenarioMap, + } + + exprFunc, _ := deleteMatchingKeys(tt.target, tt.pattern) + exprFunc(ctx) + + expected := pcommon.NewMap() + tt.want(expected) + + assert.Equal(t, expected, scenarioMap) + }) + } +} + +func Test_deleteMatchingKeys_bad_input(t *testing.T) { + input := pcommon.NewValueInt(1) + ctx := testhelper.TestTransformContext{ + Item: input, + } + + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + return ctx.GetItem() + }, + } + + exprFunc, err := deleteMatchingKeys(target, "anything") + assert.Nil(t, err) + exprFunc(ctx) + + assert.Equal(t, pcommon.NewValueInt(1), input) +} + +func Test_deleteMatchingKeys_get_nil(t *testing.T) { + ctx := testhelper.TestTransformContext{ + Item: nil, + } + + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + return ctx.GetItem() + }, + } + + exprFunc, _ := deleteMatchingKeys(target, "anything") + exprFunc(ctx) +} + +func Test_deleteMatchingKeys_invalid_pattern(t *testing.T) { + target := &testGetSetter{ + getter: func(ctx TransformContext) interface{} { + t.Errorf("nothing should be received in this scenario") + return nil + }, + } + + invalidRegexPattern := "*" + exprFunc, err := deleteMatchingKeys(target, invalidRegexPattern) + assert.Nil(t, exprFunc) + assert.Contains(t, err.Error(), "error parsing regexp:") +} diff --git a/processor/transformprocessor/internal/common/functions.go b/processor/transformprocessor/internal/common/functions.go index 2c4ab328ba39..89bd48dc9b36 100644 --- a/processor/transformprocessor/internal/common/functions.go +++ b/processor/transformprocessor/internal/common/functions.go @@ -31,6 +31,8 @@ var registry = map[string]interface{}{ "replace_all_matches": replaceAllMatches, "replace_pattern": replacePattern, "replace_all_patterns": replaceAllPatterns, + "delete_key": deleteKey, + "delete_matching_keys": deleteMatchingKeys, } type PathExpressionParser func(*Path) (GetSetter, error) diff --git a/processor/transformprocessor/internal/logs/processor_test.go b/processor/transformprocessor/internal/logs/processor_test.go index 43044b1c925e..267d50608889 100644 --- a/processor/transformprocessor/internal/logs/processor_test.go +++ b/processor/transformprocessor/internal/logs/processor_test.go @@ -109,6 +109,21 @@ func TestProcess(t *testing.T) { td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("test", "pass") }, }, + { + query: `delete_key(attributes, "http.url") where body == "operationA"`, + want: func(td plog.Logs) { + td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear() + td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.method", "get") + td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.path", "/health") + }, + }, + { + query: `delete_matching_keys(attributes, "http.*t.*") where body == "operationA"`, + want: func(td plog.Logs) { + td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear() + td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().InsertString("http.url", "http://localhost/health") + }, + }, } for _, tt := range tests { diff --git a/processor/transformprocessor/internal/metrics/processor_test.go b/processor/transformprocessor/internal/metrics/processor_test.go index 3f0ad0e1d614..4bfad34824e3 100644 --- a/processor/transformprocessor/internal/metrics/processor_test.go +++ b/processor/transformprocessor/internal/metrics/processor_test.go @@ -246,6 +246,26 @@ func TestProcess(t *testing.T) { td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(1).Attributes().InsertString("test", "pass") }, }, + { + query: []string{`delete_key(attributes, "attr3") where metric.name == "operationA"`}, + want: func(td pmetric.Metrics) { + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().Clear() + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().InsertString("attr1", "test1") + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().InsertString("attr2", "test2") + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().Clear() + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().InsertString("attr1", "test1") + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().InsertString("attr2", "test2") + }, + }, + { + query: []string{`delete_matching_keys(attributes, "[23]") where metric.name == "operationA"`}, + want: func(td pmetric.Metrics) { + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().Clear() + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().InsertString("attr1", "test1") + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().Clear() + td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().InsertString("attr1", "test1") + }, + }, } for _, tt := range tests { @@ -285,6 +305,7 @@ func fillMetricOne(m pmetric.Metric) { dataPoint0 := m.Sum().DataPoints().AppendEmpty() dataPoint0.SetStartTimestamp(StartTimestamp) + dataPoint0.SetDoubleVal(1.0) dataPoint0.Attributes().InsertString("attr1", "test1") dataPoint0.Attributes().InsertString("attr2", "test2") dataPoint0.Attributes().InsertString("attr3", "test3") diff --git a/processor/transformprocessor/internal/traces/processor_test.go b/processor/transformprocessor/internal/traces/processor_test.go index bdf1a852f3c6..d507766b06be 100644 --- a/processor/transformprocessor/internal/traces/processor_test.go +++ b/processor/transformprocessor/internal/traces/processor_test.go @@ -134,6 +134,21 @@ func TestProcess(t *testing.T) { td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().InsertString("test", "pass") }, }, + { + query: `delete_key(attributes, "http.url") where name == "operationA"`, + want: func(td ptrace.Traces) { + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().Clear() + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().InsertString("http.method", "get") + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().InsertString("http.path", "/health") + }, + }, + { + query: `delete_matching_keys(attributes, "http.*t.*") where name == "operationA"`, + want: func(td ptrace.Traces) { + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().Clear() + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().InsertString("http.url", "http://localhost/health") + }, + }, } for _, tt := range tests { From 57dfb1c33d68c0d6e14cba38c1de24bbbfa4c929 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:17:24 -0600 Subject: [PATCH 2/2] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f12723aa6eb..d986c1d76c60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - `tracegen`: support add additional resource attributes. (#11145) - `transformprocessor`: Add IsMatch factory function. This function allows regex matching in conditions (#10903) - `transformprocessor`: `replace_pattern` and `replace_all_patterns` use regex for pattern matching and replacing text in attributes/metrics. (#11125) +- `transformprocessor`: Add `delete_key` and `delete_matching_keys` which allow deleting keys from maps (#11824) - `coralogixexporter`: Add support for metrics (#11065) ### 🧰 Bug fixes 🧰