From ef38bb6d29bb48abdc0682fa0960a7938dbf1a00 Mon Sep 17 00:00:00 2001 From: Raj Nishtala <113392743+rnishtala-sumo@users.noreply.github.com> Date: Wed, 11 Oct 2023 08:23:32 -0400 Subject: [PATCH] Add an optional argument to converters to support hashing (#27235) **Description:** Functions to modify matched text during replacement can now be passed as optional arguments to the following Editors: - replace_pattern - replace_all_patterns - replace_match - replace_all_matches **Documentation:** https://github.com/rnishtala-sumo/opentelemetry-collector-contrib/blob/ottl-replace-pattern/pkg/ottl/ottlfuncs/README.md#replace_pattern **Issue:** Resolves https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/22787 --- .chloggen/ottl-replace-pattern.yaml | 32 ++++++ pkg/ottl/expression.go | 12 +- pkg/ottl/expression_test.go | 8 +- pkg/ottl/functions.go | 2 +- pkg/ottl/ottlfuncs/README.md | 62 +++++++---- .../ottlfuncs/func_replace_all_matches.go | 33 +++++- .../func_replace_all_matches_test.go | 88 ++++++++++++++- .../ottlfuncs/func_replace_all_patterns.go | 33 +++++- .../func_replace_all_patterns_test.go | 105 +++++++++++++++++- pkg/ottl/ottlfuncs/func_replace_match.go | 32 +++++- pkg/ottl/ottlfuncs/func_replace_match_test.go | 95 +++++++++++++++- pkg/ottl/ottlfuncs/func_replace_pattern.go | 33 +++++- .../ottlfuncs/func_replace_pattern_test.go | 100 ++++++++++++++++- 13 files changed, 565 insertions(+), 70 deletions(-) create mode 100755 .chloggen/ottl-replace-pattern.yaml diff --git a/.chloggen/ottl-replace-pattern.yaml b/.chloggen/ottl-replace-pattern.yaml new file mode 100755 index 000000000000..4e85584d1439 --- /dev/null +++ b/.chloggen/ottl-replace-pattern.yaml @@ -0,0 +1,32 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add optional Converter parameters to replacement Editors + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [27235] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Functions to modify matched text during replacement can now be passed as optional arguments to the following Editors: + - `replace_pattern` + - `replace_all_patterns` + - `replace_match` + - `replace_all_matches` + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/pkg/ottl/expression.go b/pkg/ottl/expression.go index 18e5ffad56ce..e826d4324055 100644 --- a/pkg/ottl/expression.go +++ b/pkg/ottl/expression.go @@ -254,8 +254,8 @@ type FunctionGetter[K any] interface { // StandardFunctionGetter is a basic implementation of FunctionGetter. type StandardFunctionGetter[K any] struct { - fCtx FunctionContext - fact Factory[K] + FCtx FunctionContext + Fact Factory[K] } // Get takes an Arguments struct containing arguments the caller wants passed to the @@ -263,12 +263,12 @@ type StandardFunctionGetter[K any] struct { // If there is a mismatch between the function's signature and the arguments the caller // wants to pass to the function, an error is returned. func (g StandardFunctionGetter[K]) Get(args Arguments) (Expr[K], error) { - if g.fact == nil { + if g.Fact == nil { return Expr[K]{}, fmt.Errorf("undefined function") } - fArgs := g.fact.CreateDefaultArguments() + fArgs := g.Fact.CreateDefaultArguments() if reflect.TypeOf(fArgs).Kind() != reflect.Pointer { - return Expr[K]{}, fmt.Errorf("factory for %q must return a pointer to an Arguments value in its CreateDefaultArguments method", g.fact.Name()) + return Expr[K]{}, fmt.Errorf("factory for %q must return a pointer to an Arguments value in its CreateDefaultArguments method", g.Fact.Name()) } if reflect.TypeOf(args).Kind() != reflect.Pointer { return Expr[K]{}, fmt.Errorf("%q must be pointer to an Arguments value", reflect.TypeOf(args).Kind()) @@ -282,7 +282,7 @@ func (g StandardFunctionGetter[K]) Get(args Arguments) (Expr[K], error) { field := argsVal.Field(i) fArgsVal.Field(i).Set(field) } - fn, err := g.fact.CreateFunction(g.fCtx, fArgs) + fn, err := g.Fact.CreateFunction(g.FCtx, fArgs) if err != nil { return Expr[K]{}, fmt.Errorf("couldn't create function: %w", err) } diff --git a/pkg/ottl/expression_test.go b/pkg/ottl/expression_test.go index 2324db49d619..7b57713fb2c7 100644 --- a/pkg/ottl/expression_test.go +++ b/pkg/ottl/expression_test.go @@ -720,7 +720,7 @@ func Test_FunctionGetter(t *testing.T) { return "str", nil }, }, - function: StandardFunctionGetter[any]{fCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, fact: functions["SHA256"]}, + function: StandardFunctionGetter[any]{FCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, Fact: functions["SHA256"]}, want: "anything", valid: true, }, @@ -731,7 +731,7 @@ func Test_FunctionGetter(t *testing.T) { return nil, nil }, }, - function: StandardFunctionGetter[any]{fCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, fact: functions["SHA250"]}, + function: StandardFunctionGetter[any]{FCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, Fact: functions["SHA250"]}, want: "anything", valid: false, expectedErrorMsg: "undefined function", @@ -743,7 +743,7 @@ func Test_FunctionGetter(t *testing.T) { return nil, nil }, }, - function: StandardFunctionGetter[any]{fCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, fact: functions["test_arg_mismatch"]}, + function: StandardFunctionGetter[any]{FCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, Fact: functions["test_arg_mismatch"]}, want: "anything", valid: false, expectedErrorMsg: "incorrect number of arguments. Expected: 4 Received: 1", @@ -755,7 +755,7 @@ func Test_FunctionGetter(t *testing.T) { return nil, nil }, }, - function: StandardFunctionGetter[any]{fCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, fact: functions["cannot_create_function"]}, + function: StandardFunctionGetter[any]{FCtx: FunctionContext{Set: componenttest.NewNopTelemetrySettings()}, Fact: functions["cannot_create_function"]}, want: "anything", valid: false, expectedErrorMsg: "couldn't create function: error", diff --git a/pkg/ottl/functions.go b/pkg/ottl/functions.go index fbe2120b6ee0..37aab7e1c7ca 100644 --- a/pkg/ottl/functions.go +++ b/pkg/ottl/functions.go @@ -123,7 +123,7 @@ func (p *Parser[K]) buildArgs(ed editor, argsVal reflect.Value) error { if !ok { return fmt.Errorf("undefined function %s", name) } - val = StandardFunctionGetter[K]{fCtx: FunctionContext{Set: p.telemetrySettings}, fact: f} + val = StandardFunctionGetter[K]{FCtx: FunctionContext{Set: p.telemetrySettings}, Fact: f} case fieldType.Kind() == reflect.Slice: val, err = p.buildSliceArg(arg.Value, fieldType) default: diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index eda7cb703f80..47c7fc7460de 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -16,6 +16,7 @@ Some functions are able to handle different types and will generally convert tho In these situations the function will error if it does not know how to do the conversion. Use `ErrorMode` to determine how the `Statement` handles these errors. See the component-specific guides for how each uses error mode: + - [filterprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor#ottl) - [routingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/routingprocessor#tech-preview-opentelemetry-transformation-language-statements-as-routing-conditions) - [transformprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor#config) @@ -26,11 +27,12 @@ Editors are what OTTL uses to transform telemetry. Editors: -- Are allowed to transform telemetry. When a Function is invoked the expectation is that the underlying telemetry is modified in some way. -- May have side effects. Some Functions may generate telemetry and add it to the telemetry payload to be processed in this batch. -- May return values. Although not common and not required, Functions may return values. +- Are allowed to transform telemetry. When a Function is invoked the expectation is that the underlying telemetry is modified in some way. +- May have side effects. Some Functions may generate telemetry and add it to the telemetry payload to be processed in this batch. +- May return values. Although not common and not required, Functions may return values. Available Editors: + - [delete_key](#delete_key) - [delete_matching_keys](#delete_matching_keys) - [keep_keys](#keep_keys) @@ -55,8 +57,8 @@ The key will be deleted from the map. Examples: -- `delete_key(attributes, "http.request.header.authorization")` +- `delete_key(attributes, "http.request.header.authorization")` - `delete_key(resource.attributes, "http.request.header.authorization")` @@ -72,8 +74,8 @@ All keys that match the pattern will be deleted from the map. Examples: -- `delete_key(attributes, "http.request.header.authorization")` +- `delete_key(attributes, "http.request.header.authorization")` - `delete_key(resource.attributes, "http.request.header.authorization")` @@ -126,6 +128,7 @@ The `merge_maps` function merges the source map into the target map using the su `target` is a `pdata.Map` type field. `source` is a `pdata.Map` type field. `strategy` is a string that must be one of `insert`, `update`, or `upsert`. If strategy is: + - `insert`: Insert the value from `source` into `target` where the key does not already exist. - `update`: Update the entry in `target` with the value from `source` where the key does exist. - `upsert`: Performs insert or update. Insert the value from `source` into `target` where the key does not already exist and update the entry in `target` with the value from `source` where the key does exist. @@ -144,11 +147,11 @@ Examples: ### replace_all_matches -`replace_all_matches(target, pattern, replacement)` +`replace_all_matches(target, pattern, replacement, function)` The `replace_all_matches` function replaces any matching string value with the replacement string. -`target` is a path expression to a `pdata.Map` type field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. +`target` is a path expression to a `pdata.Map` type field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching string with the hash value of `replacement`. Each string value in `target` that matches `pattern` will get replaced with `replacement`. Non-string values are ignored. @@ -158,10 +161,11 @@ There is currently a bug with OTTL that does not allow the pattern to end with ` Examples: - `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}")` +- `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256)` ### replace_all_patterns -`replace_all_patterns(target, mode, regex, replacement)` +`replace_all_patterns(target, mode, regex, replacement, function)` The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -173,6 +177,8 @@ If one or more sections of `target` match `regex` they will get replaced with `r The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. + There is currently a bug with OTTL that does not allow the pattern to end with `\\"`. If your pattern needs to end with backslashes, add something inconsequential to the end of the pattern such as `{1}`, `$`, or `.*`. [See Issue 23238 for details](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23238). @@ -182,6 +188,7 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -189,7 +196,7 @@ If using OTTL outside of collector configuration, `$` should not be escaped and ### replace_match -`replace_match(target, pattern, replacement)` +`replace_match(target, pattern, replacement, function)` The `replace_match` function allows replacing entire strings if they match a glob pattern. @@ -197,16 +204,19 @@ The `replace_match` function allows replacing entire strings if they match a glo If `target` matches `pattern` it will get replaced with `replacement`. +The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching glob pattern with the hash value of `replacement`. + There is currently a bug with OTTL that does not allow the pattern to end with `\\"`. [See Issue 23238 for details](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23238). Examples: - `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}")` +- `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256)` ### replace_pattern -`replace_pattern(target, regex, replacement)` +`replace_pattern(target, regex, replacement, function)` The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -216,6 +226,8 @@ If one or more sections of `target` match `regex` they will get replaced with `r The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. + There is currently a bug with OTTL that does not allow the pattern to end with `\\"`. If your pattern needs to end with backslashes, add something inconsequential to the end of the pattern such as `{1}`, `$`, or `.*`. [See Issue 23238 for details](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23238). @@ -224,6 +236,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -275,6 +288,7 @@ Converters are pure functions that take OTTL values as input and output a single Unlike functions, they do not modify any input telemetry and always return a value. Available Converters: + - [Concat](#concat) - [ConvertCase](#convertcase) - [ExtractPatterns](#extractpatterns) @@ -373,9 +387,9 @@ Examples: The `ExtractPatterns` Converter returns a `pcommon.Map` struct that is a result of extracting named capture groups from the target string. If not matches are found then an empty `pcommon.Map` is returned. -`target` is a Getter that returns a string. `pattern` is a regex string. +`target` is a Getter that returns a string. `pattern` is a regex string. -If `target` is not a string or nil `ExtractPatterns` will return an error. If `pattern` does not contain at least 1 named capture group then `ExtractPatterns` will error on startup. +If `target` is not a string or nil `ExtractPatterns` will return an error. If `pattern` does not contain at least 1 named capture group then `ExtractPatterns` will error on startup. Examples: @@ -425,10 +439,11 @@ The `Int` Converter converts the `value` to int type. The returned type is int64. The input `value` types: -* float64. Fraction is discharged (truncation towards zero). -* string. Trying to parse an integer from string if it fails then nil will be returned. -* bool. If `value` is true, then the function will return 1 otherwise 0. -* int64. The function returns the `value` without changes. + +- float64. Fraction is discharged (truncation towards zero). +- string. Trying to parse an integer from string if it fails then nil will be returned. +- bool. If `value` is true, then the function will return 1 otherwise 0. +- int64. The function returns the `value` without changes. If `value` is another type or parsing failed nil is always returned. @@ -531,8 +546,8 @@ If target is not a float64, it will be converted to one: - int64s are converted to float64s - strings are converted using `strconv` -- booleans are converted using `1` for `true` and `0` for `false`. This means passing `false` to the function will cause an error. -- int, float, string, and bool OTLP Values are converted following the above rules depending on their type. Other types cause an error. +- booleans are converted using `1` for `true` and `0` for `false`. This means passing `false` to the function will cause an error. +- int, float, string, and bool OTLP Values are converted following the above rules depending on their type. Other types cause an error. If target is nil an error is returned. @@ -713,7 +728,7 @@ Examples: ### Split -`Split(target, delimiter)` +```Split(target, delimiter)``` The `Split` Converter separates a string by the delimiter, and returns an array of substrings. @@ -726,7 +741,7 @@ There is currently a bug with OTTL that does not allow the target string to end Examples: -- ```Split("A|B|C", "|")``` +- `Split("A|B|C", "|")` ### Substring @@ -773,7 +788,7 @@ Examples: The `TruncateTime` Converter returns the given time rounded down to a multiple of the given duration. The Converter [uses the `time.Truncate` function](https://pkg.go.dev/time#Time.Truncate). -`time` is a `time.Time`. `duration` is a `time.Duration`. If `time` is not a `time.Time` or if `duration` is not a `time.Duration`, an error will be returned. +`time` is a `time.Time`. `duration` is a `time.Duration`. If `time` is not a `time.Time` or if `duration` is not a `time.Duration`, an error will be returned. While some common paths can return a `time.Time` object, you will most like need to use the [Duration Converter](#duration) to create a `time.Duration`. @@ -846,9 +861,10 @@ The `UUID` function generates a v4 uuid string. ## Function syntax Functions should be named and formatted according to the following standards. + - Function names MUST start with a verb unless it is a Factory that creates a new type. - Converters MUST be UpperCamelCase. - Function names that contain multiple words MUST separate those words with `_`. -- Functions that interact with multiple items MUST have plurality in the name. Ex: `truncate_all`, `keep_keys`, `replace_all_matches`. -- Functions that interact with a single item MUST NOT have plurality in the name. If a function would interact with multiple items due to a condition, like `where`, it is still considered singular. Ex: `set`, `delete`, `replace_match`. +- Functions that interact with multiple items MUST have plurality in the name. Ex: `truncate_all`, `keep_keys`, `replace_all_matches`. +- Functions that interact with a single item MUST NOT have plurality in the name. If a function would interact with multiple items due to a condition, like `where`, it is still considered singular. Ex: `set`, `delete`, `replace_match`. - Functions that change a specific target MUST set the target as the first parameter. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_matches.go b/pkg/ottl/ottlfuncs/func_replace_all_matches.go index 4a686be2e958..8ea90b32fe0f 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_matches.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_matches.go @@ -17,6 +17,11 @@ type ReplaceAllMatchesArguments[K any] struct { Target ottl.PMapGetter[K] Pattern string Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] +} + +type replaceAllMatchesFuncArgs[K any] struct { + Input ottl.StringGetter[K] } func NewReplaceAllMatchesFactory[K any]() ottl.Factory[K] { @@ -30,22 +35,40 @@ func createReplaceAllMatchesFunction[K any](_ ottl.FunctionContext, oArgs ottl.A return nil, fmt.Errorf("ReplaceAllMatchesFactory args must be of type *ReplaceAllMatchesArguments[K]") } - return replaceAllMatches(args.Target, args.Pattern, args.Replacement) + return replaceAllMatches(args.Target, args.Pattern, args.Replacement, args.Function) } -func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replacement ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { +func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) } return func(ctx context.Context, tCtx K) (interface{}, error) { val, err := target.Get(ctx, tCtx) + var replacementVal string if err != nil { return nil, err } - replacementVal, err := replacement.Get(ctx, tCtx) - if err != nil { - return nil, err + if fn.IsEmpty() { + replacementVal, err = replacement.Get(ctx, tCtx) + if err != nil { + return nil, err + } + } else { + fnVal := fn.Get() + replacementExpr, errNew := fnVal.Get(&replaceAllMatchesFuncArgs[K]{Input: replacement}) + if errNew != nil { + return nil, errNew + } + replacementValRaw, errNew := replacementExpr.Eval(ctx, tCtx) + if errNew != nil { + return nil, errNew + } + replacementValStr, ok := replacementValRaw.(string) + if !ok { + return nil, fmt.Errorf("replacement value is not a string") + } + replacementVal = replacementValStr } val.Range(func(key string, value pcommon.Value) bool { if glob.Match(value.Str()) { diff --git a/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go b/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go index d2690bb1c1d3..4529f701ebc3 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" @@ -19,6 +21,14 @@ func Test_replaceAllMatches(t *testing.T) { input.PutStr("test2", "hello") input.PutStr("test3", "goodbye") + ottlValue := ottl.StandardFunctionGetter[pcommon.Map]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Map]()["SHA256"], + } + optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Map]](ottlValue) + target := &ottl.StandardPMapGetter[pcommon.Map]{ Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { return tCtx, nil @@ -30,8 +40,25 @@ func Test_replaceAllMatches(t *testing.T) { target ottl.PMapGetter[pcommon.Map] pattern string replacement ottl.StringGetter[pcommon.Map] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] want func(pcommon.Map) }{ + { + name: "replace only matches (with hash function)", + target: target, + pattern: "hello*", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (interface{}, error) { + return "hello {universe}", nil + }, + }, + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") + expectedMap.PutStr("test2", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") + expectedMap.PutStr("test3", "goodbye") + }, + }, { name: "replace only matches", target: target, @@ -41,6 +68,7 @@ func Test_replaceAllMatches(t *testing.T) { return "hello {universe}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe}") expectedMap.PutStr("test2", "hello {universe}") @@ -56,6 +84,7 @@ func Test_replaceAllMatches(t *testing.T) { return "nothing {matches}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -68,7 +97,7 @@ func Test_replaceAllMatches(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllMatches(tt.target, tt.pattern, tt.replacement) + exprFunc, err := replaceAllMatches(tt.target, tt.pattern, tt.replacement, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioMap) @@ -95,13 +124,65 @@ func Test_replaceAllMatches_bad_input(t *testing.T) { return "{replacement}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllMatches[interface{}](target, "*", replacement) + exprFunc, err := replaceAllMatches[interface{}](target, "*", replacement, function) assert.NoError(t, err) _, err = exprFunc(nil, input) assert.Error(t, err) } +func Test_replaceAllMatches_bad_function_input(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardPMapGetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return nil, nil + }, + } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} + + exprFunc, err := replaceAllMatches[interface{}](target, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "expected pcommon.Map") + assert.Nil(t, result) +} + +func Test_replaceAllMatches_bad_function_result(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardPMapGetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{replacement}", nil + }, + } + ottlValue := ottl.StandardFunctionGetter[interface{}]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[interface{}]()["IsString"], + } + function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) + + exprFunc, err := replaceAllMatches[interface{}](target, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.Nil(t, result) +} + func Test_replaceAllMatches_get_nil(t *testing.T) { target := &ottl.StandardPMapGetter[interface{}]{ Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { @@ -113,8 +194,9 @@ func Test_replaceAllMatches_get_nil(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllMatches[interface{}](target, "*", replacement) + exprFunc, err := replaceAllMatches[interface{}](target, "*", replacement, function) assert.NoError(t, err) _, err = exprFunc(nil, nil) assert.Error(t, err) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index c583f00a34ba..47021b6bff01 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -23,6 +23,11 @@ type ReplaceAllPatternsArguments[K any] struct { Mode string RegexPattern string Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] +} + +type replaceAllPatternFuncArgs[K any] struct { + Input ottl.StringGetter[K] } func NewReplaceAllPatternsFactory[K any]() ottl.Factory[K] { @@ -36,10 +41,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) @@ -50,12 +55,30 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt return func(ctx context.Context, tCtx K) (interface{}, error) { val, err := target.Get(ctx, tCtx) + var replacementVal string if err != nil { return nil, err } - replacementVal, err := replacement.Get(ctx, tCtx) - if err != nil { - return nil, err + if fn.IsEmpty() { + replacementVal, err = replacement.Get(ctx, tCtx) + if err != nil { + return nil, err + } + } else { + fnVal := fn.Get() + replacementExpr, errNew := fnVal.Get(&replaceAllPatternFuncArgs[K]{Input: replacement}) + if errNew != nil { + return nil, errNew + } + replacementValRaw, errNew := replacementExpr.Eval(ctx, tCtx) + if errNew != nil { + return nil, errNew + } + replacementValStr, ok := replacementValRaw.(string) + if !ok { + return nil, fmt.Errorf("replacement value is not a string") + } + replacementVal = replacementValStr } updated := pcommon.NewMap() updated.EnsureCapacity(val.Len()) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index 58ad07893d6f..29bb2b2669f0 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" @@ -23,6 +24,14 @@ func Test_replaceAllPatterns(t *testing.T) { input.PutDouble("test5", 1234) input.PutBool("test6", true) + ottlValue := ottl.StandardFunctionGetter[pcommon.Map]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Map]()["SHA256"], + } + optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Map]](ottlValue) + target := &ottl.StandardPMapGetter[pcommon.Map]{ Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { return tCtx, nil @@ -35,8 +44,29 @@ func Test_replaceAllPatterns(t *testing.T) { mode string pattern string replacement ottl.StringGetter[pcommon.Map] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] want func(pcommon.Map) }{ + { + name: "replace only matches (with hash function)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (interface{}, error) { + return "hello {universe}", nil + }, + }, + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad world") + expectedMap.PutStr("test2", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") + expectedMap.PutStr("test3", "goodbye world1 and world2") + expectedMap.PutInt("test4", 1234) + expectedMap.PutDouble("test5", 1234) + expectedMap.PutBool("test6", true) + }, + }, { name: "replace only matches", target: target, @@ -47,6 +77,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") expectedMap.PutStr("test2", "hello {universe}") @@ -66,6 +97,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -85,6 +117,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") expectedMap.PutStr("test2", "hello") @@ -104,6 +137,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -124,6 +158,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -144,6 +179,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test.", "hello world") @@ -164,6 +200,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -184,6 +221,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test-2", "hello") @@ -203,6 +241,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -219,7 +258,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) @@ -235,7 +274,6 @@ func Test_replaceAllPatterns(t *testing.T) { func Test_replaceAllPatterns_bad_input(t *testing.T) { input := pcommon.NewValueStr("not a map") - target := &ottl.StandardPMapGetter[interface{}]{ Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { return tCtx, nil @@ -246,14 +284,66 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { return "{replacement}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexpattern", replacement) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexpattern", replacement, function) assert.Nil(t, err) _, err = exprFunc(nil, input) assert.Error(t, err) } +func Test_replaceAllPatterns_bad_function_input(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardPMapGetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return nil, nil + }, + } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} + + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "expected pcommon.Map") + assert.Nil(t, result) +} + +func Test_replaceAllPatterns_bad_function_result(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardPMapGetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{anything}", nil + }, + } + ottlValue := ottl.StandardFunctionGetter[interface{}]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[interface{}]()["IsString"], + } + function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) + + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.Nil(t, result) +} + func Test_replaceAllPatterns_get_nil(t *testing.T) { target := &ottl.StandardPMapGetter[interface{}]{ Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { @@ -265,8 +355,9 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, function) assert.NoError(t, err) _, err = exprFunc(nil, nil) @@ -285,9 +376,10 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidRegexPattern := "*" - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, replacement) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, replacement, function) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") assert.Nil(t, exprFunc) @@ -305,9 +397,10 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidMode := "invalid" - exprFunc, err := replaceAllPatterns[interface{}](target, invalidMode, "regex", replacement) + exprFunc, err := replaceAllPatterns[interface{}](target, invalidMode, "regex", replacement, function) assert.Nil(t, exprFunc) assert.Contains(t, err.Error(), "invalid mode") } diff --git a/pkg/ottl/ottlfuncs/func_replace_match.go b/pkg/ottl/ottlfuncs/func_replace_match.go index 578e84becfff..d56ef4b75edd 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match.go +++ b/pkg/ottl/ottlfuncs/func_replace_match.go @@ -16,6 +16,11 @@ type ReplaceMatchArguments[K any] struct { Target ottl.GetSetter[K] Pattern string Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] +} + +type replaceMatchFuncArgs[K any] struct { + Input ottl.StringGetter[K] } func NewReplaceMatchFactory[K any]() ottl.Factory[K] { @@ -29,20 +34,41 @@ func createReplaceMatchFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argume return nil, fmt.Errorf("ReplaceMatchFactory args must be of type *ReplaceMatchArguments[K]") } - return replaceMatch(args.Target, args.Pattern, args.Replacement) + return replaceMatch(args.Target, args.Pattern, args.Replacement, args.Function) } -func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { +func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) } return func(ctx context.Context, tCtx K) (interface{}, error) { val, err := target.Get(ctx, tCtx) + var replacementVal string if err != nil { return nil, err } - replacementVal, err := replacement.Get(ctx, tCtx) + if fn.IsEmpty() { + replacementVal, err = replacement.Get(ctx, tCtx) + if err != nil { + return nil, err + } + } else { + fnVal := fn.Get() + replacementExpr, errNew := fnVal.Get(&replaceMatchFuncArgs[K]{Input: replacement}) + if errNew != nil { + return nil, errNew + } + replacementValRaw, errNew := replacementExpr.Eval(ctx, tCtx) + if errNew != nil { + return nil, errNew + } + replacementValStr, ok := replacementValRaw.(string) + if !ok { + return nil, fmt.Errorf("replacement value is not a string") + } + replacementVal = replacementValStr + } if err != nil { return nil, err } diff --git a/pkg/ottl/ottlfuncs/func_replace_match_test.go b/pkg/ottl/ottlfuncs/func_replace_match_test.go index b2b726bf51fd..e3fe48cca750 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_match_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" @@ -15,7 +17,13 @@ import ( func Test_replaceMatch(t *testing.T) { input := pcommon.NewValueStr("hello world") - + ottlValue := ottl.StandardFunctionGetter[pcommon.Value]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Value]()["SHA256"], + } + optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) target := &ottl.StandardGetSetter[pcommon.Value]{ Getter: func(ctx context.Context, tCtx pcommon.Value) (interface{}, error) { return tCtx.Str(), nil @@ -31,8 +39,23 @@ func Test_replaceMatch(t *testing.T) { target ottl.GetSetter[pcommon.Value] pattern string replacement ottl.StringGetter[pcommon.Value] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] want func(pcommon.Value) }{ + { + name: "replace match (with hash function)", + target: target, + pattern: "hello*", + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "hello {universe}", nil + }, + }, + function: optionalArg, + want: func(expectedValue pcommon.Value) { + expectedValue.SetStr("4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") + }, + }, { name: "replace match", target: target, @@ -42,6 +65,7 @@ func Test_replaceMatch(t *testing.T) { return "hello {universe}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello {universe}") }, @@ -55,6 +79,7 @@ func Test_replaceMatch(t *testing.T) { return "goodbye {universe}", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello world") }, @@ -64,7 +89,7 @@ func Test_replaceMatch(t *testing.T) { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replaceMatch(tt.target, tt.pattern, tt.replacement) + exprFunc, err := replaceMatch(tt.target, tt.pattern, tt.replacement, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) assert.NoError(t, err) @@ -94,8 +119,9 @@ func Test_replaceMatch_bad_input(t *testing.T) { return "{replacement}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceMatch[interface{}](target, "*", replacement) + exprFunc, err := replaceMatch[interface{}](target, "*", replacement, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -105,6 +131,66 @@ func Test_replaceMatch_bad_input(t *testing.T) { assert.Equal(t, pcommon.NewValueInt(1), input) } +func Test_replaceMatch_bad_function_input(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardGetSetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + Setter: func(ctx context.Context, tCtx interface{}, val interface{}) error { + t.Errorf("nothing should be set in this scenario") + return nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return nil, nil + }, + } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} + + exprFunc, err := replaceMatch[interface{}](target, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "expected string but got nil") + assert.Nil(t, result) +} + +func Test_replaceMatch_bad_function_result(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardGetSetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + Setter: func(ctx context.Context, tCtx interface{}, val interface{}) error { + t.Errorf("nothing should be set in this scenario") + return nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return nil, nil + }, + } + ottlValue := ottl.StandardFunctionGetter[interface{}]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[interface{}]()["IsString"], + } + function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) + + exprFunc, err := replaceMatch[interface{}](target, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "replacement value is not a string") + assert.Nil(t, result) +} + func Test_replaceMatch_get_nil(t *testing.T) { target := &ottl.StandardGetSetter[interface{}]{ Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { @@ -120,8 +206,9 @@ func Test_replaceMatch_get_nil(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceMatch[interface{}](target, "*", replacement) + exprFunc, err := replaceMatch[interface{}](target, "*", replacement, function) assert.NoError(t, err) result, err := exprFunc(nil, nil) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 1386fec80c01..b69ea098e2fa 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -15,6 +15,11 @@ type ReplacePatternArguments[K any] struct { Target ottl.GetSetter[K] RegexPattern string Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] +} + +type replacePatternFuncArgs[K any] struct { + Input ottl.StringGetter[K] } func NewReplacePatternFactory[K any]() ottl.Factory[K] { @@ -28,22 +33,40 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function) } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) } return func(ctx context.Context, tCtx K) (interface{}, error) { originalVal, err := target.Get(ctx, tCtx) + var replacementVal string if err != nil { return nil, err } - replacementVal, err := replacement.Get(ctx, tCtx) - if err != nil { - return nil, err + if fn.IsEmpty() { + replacementVal, err = replacement.Get(ctx, tCtx) + if err != nil { + return nil, err + } + } else { + fnVal := fn.Get() + replacementExpr, errNew := fnVal.Get(&replacePatternFuncArgs[K]{Input: replacement}) + if errNew != nil { + return nil, errNew + } + replacementValRaw, errNew := replacementExpr.Eval(ctx, tCtx) + if errNew != nil { + return nil, errNew + } + replacementValStr, ok := replacementValRaw.(string) + if !ok { + return nil, fmt.Errorf("replacement value is not a string") + } + replacementVal = replacementValStr } if originalVal == nil { return nil, nil diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 278cc2ccbb57..87812b2684ac 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" @@ -16,7 +17,13 @@ import ( func Test_replacePattern(t *testing.T) { input := pcommon.NewValueStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") - + ottlValue := ottl.StandardFunctionGetter[pcommon.Value]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Value]()["SHA256"], + } + optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) target := &ottl.StandardGetSetter[pcommon.Value]{ Getter: func(ctx context.Context, tCtx pcommon.Value) (interface{}, error) { return tCtx.Str(), nil @@ -32,8 +39,23 @@ func Test_replacePattern(t *testing.T) { target ottl.GetSetter[pcommon.Value] pattern string replacement ottl.StringGetter[pcommon.Value] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] want func(pcommon.Value) }{ + { + name: "replace regex match (with hash function)", + target: target, + pattern: `passwd\=[^\s]*(\s?)`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=*** ", nil + }, + }, + function: optionalArg, + want: func(expectedValue pcommon.Value) { + expectedValue.SetStr("application 0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838beotherarg=notsensitive key1 key2") + }, + }, { name: "replace regex match", target: target, @@ -43,6 +65,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") }, @@ -56,6 +79,7 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") }, @@ -69,6 +93,7 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") }, @@ -82,6 +107,7 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") }, @@ -95,6 +121,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") }, @@ -103,7 +130,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -134,8 +161,9 @@ func Test_replacePattern_bad_input(t *testing.T) { return "{replacement}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -144,6 +172,66 @@ func Test_replacePattern_bad_input(t *testing.T) { assert.Equal(t, pcommon.NewValueInt(1), input) } +func Test_replacePattern_bad_function_input(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardGetSetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + Setter: func(ctx context.Context, tCtx interface{}, val interface{}) error { + t.Errorf("nothing should be set in this scenario") + return nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return nil, nil + }, + } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} + + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "expected string but got nil") + assert.Nil(t, result) +} + +func Test_replacePattern_bad_function_result(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardGetSetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return tCtx, nil + }, + Setter: func(ctx context.Context, tCtx interface{}, val interface{}) error { + t.Errorf("nothing should be set in this scenario") + return nil + }, + } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return nil, nil + }, + } + ottlValue := ottl.StandardFunctionGetter[interface{}]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[interface{}]()["IsString"], + } + function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) + + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, function) + assert.NoError(t, err) + + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "replacement value is not a string") + assert.Nil(t, result) +} + func Test_replacePattern_get_nil(t *testing.T) { target := &ottl.StandardGetSetter[interface{}]{ Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { @@ -159,8 +247,9 @@ func Test_replacePattern_get_nil(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement) + exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement, function) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -184,9 +273,10 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { return "{anything}", nil }, } + function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidRegexPattern := "*" - _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement) + _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement, function) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") }