diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a9ea1..8d94472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added +- Added ability to adjust merging behavior based on field names in configuration. Using `ucfg.FieldMergeValues`, `ucfg.FieldReplaceValues`, `ucfg.FieldAppendValues`, and `ucfg.FieldPrependValues`. #151 ### Changed diff --git a/merge.go b/merge.go index fa2dd07..ff07a97 100644 --- a/merge.go +++ b/merge.go @@ -115,6 +115,10 @@ func mergeConfigDict(opts *options, to, from *Config) Error { } old, _ := to.fields.get(k) + opts, err := fieldOptsOverride(opts, k, -1) + if err != nil { + return err + } merged, err := mergeValues(opts, old, v) if err != nil { return err @@ -128,7 +132,12 @@ func mergeConfigDict(opts *options, to, from *Config) Error { } func mergeConfigArr(opts *options, to, from *Config) Error { - switch opts.configValueHandling { + currHandling := opts.configValueHandling + opts, err := fieldOptsOverride(opts, "*", -1) + if err != nil { + return err + } + switch currHandling { case cfgReplaceValue: return mergeConfigReplaceArr(opts, to, from) @@ -177,8 +186,13 @@ func mergeConfigMergeArr(opts *options, to, from *Config) Error { field: fmt.Sprintf("%v", i), } + // possible for individual index to be replaced + idxOpts, err := fieldOptsOverride(opts, "", i) + if err != nil { + return err + } old := to.fields.array()[i] - merged, err := mergeValues(opts, old, arr[i]) + merged, err := mergeValues(idxOpts, old, arr[i]) if err != nil { return err } @@ -515,3 +529,57 @@ func normalizeString(ctx context, opts *options, str string) (value, Error) { return newSplice(ctx, opts.meta, varexp), nil } + +func fieldOptsOverride(opts *options, fieldName string, idx int) (*options, Error) { + if opts.fieldHandlingTree == nil { + return opts, nil + } + cfgHandling, child, ok := opts.fieldHandlingTree.fieldHandling(fieldName, idx) + child, err := includeWildcard(child, opts.fieldHandlingTree) + if err != nil { + return nil, err + } + if !ok { + // Only return a new `options` when arriving at new nested child. This + // combined with optimizations in `includeWildcard` will ensure that only + // a new opts will be created and returned when absolutely required. + if child != nil && opts.fieldHandlingTree != child { + newOpts := *opts + newOpts.fieldHandlingTree = child + opts = &newOpts + } + return opts, nil + } + // Only return a new `options` if absolutely required. + if opts.configValueHandling != cfgHandling || opts.fieldHandlingTree != child { + newOpts := *opts + newOpts.configValueHandling = cfgHandling + newOpts.fieldHandlingTree = child + opts = &newOpts + } + return opts, nil +} + +func includeWildcard(child *fieldHandlingTree, parent *fieldHandlingTree) (*fieldHandlingTree, Error) { + if parent == nil { + return child, nil + } + wildcard, err := parent.wildcard() + if err != nil { + return child, nil + } + if child == nil && len(parent.fields.dict()) == 1 { + // parent is already config with just wildcard + return parent, nil + } + sub := newFieldHandlingTree() + if child != nil { + if err := sub.merge(child); err != nil { + return nil, err.(Error) + } + } + if err := sub.setWildcard(wildcard); err != nil { + return nil, err.(Error) + } + return sub, nil +} diff --git a/merge_test.go b/merge_test.go index 2d89782..d1339b2 100644 --- a/merge_test.go +++ b/merge_test.go @@ -390,6 +390,495 @@ func TestMergeChildArray(t *testing.T) { } } +func TestMergeFieldHandling(t *testing.T) { + + tests := []struct { + Name string + Configs []interface{} + Options []Option + Assert func(t *testing.T, c *Config) + }{ + { + "default append w/ replace paths", + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + AppendValues, + FieldReplaceValues("paths"), + }, + func(t *testing.T, c *Config) { + unpacked := make(map[string]interface{}) + assert.NoError(t, c.Unpack(unpacked)) + + paths, _ := unpacked["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := unpacked["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_locale", "add_fields"}, processorNames) + }, + }, + { + "default prepend w/ replace paths", + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + PrependValues, + FieldReplaceValues("paths"), + }, + func(t *testing.T, c *Config) { + unpacked := make(map[string]interface{}) + assert.NoError(t, c.Unpack(unpacked)) + + paths, _ := unpacked["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := unpacked["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_fields", "add_locale"}, processorNames) + }, + }, + { + "replace paths and append processors", + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + FieldReplaceValues("paths"), + FieldAppendValues("processors"), + }, + func(t *testing.T, c *Config) { + unpacked := make(map[string]interface{}) + assert.NoError(t, c.Unpack(unpacked)) + + paths, _ := unpacked["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := unpacked["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_locale", "add_fields"}, processorNames) + }, + }, + { + "default append w/ replace paths and prepend processors", + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + AppendValues, + FieldReplaceValues("paths"), + FieldPrependValues("processors"), + }, + func(t *testing.T, c *Config) { + unpacked := make(map[string]interface{}) + assert.NoError(t, c.Unpack(unpacked)) + + paths, _ := unpacked["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := unpacked["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_fields", "add_locale"}, processorNames) + }, + }, + { + "nested replace paths and append processors", + []interface{}{ + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + }, + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + FieldReplaceValues("*.paths"), + FieldAppendValues("*.processors"), + }, + func(t *testing.T, c *Config) { + var unpacked []interface{} + assert.NoError(t, c.Unpack(&unpacked)) + + nested := unpacked[0].(map[string]interface{}) + paths, _ := nested["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := nested["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_locale", "add_fields"}, processorNames) + }, + }, + { + "deep unknown nested replace paths and append processors", + []interface{}{ + []interface{}{ + map[string]interface{}{ + "deep": []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + }, + }, + }, + []interface{}{ + map[string]interface{}{ + "deep": []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + FieldReplaceValues("**.paths"), + FieldAppendValues("**.processors"), + }, + func(t *testing.T, c *Config) { + var unpacked []interface{} + assert.NoError(t, c.Unpack(&unpacked)) + + level0 := unpacked[0].(map[string]interface{}) + deep, _ := level0["deep"].([]interface{}) + nested := deep[0].(map[string]interface{}) + paths, _ := nested["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := nested["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_locale", "add_fields"}, processorNames) + }, + }, + { + "replace paths and append processors using depth selector (but fields are at level0)", + []interface{}{ + map[string]interface{}{ + "paths": []interface{}{ + "removed_1.log", + "removed_2.log", + "removed_2.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + }, + }, + map[string]interface{}{ + "paths": []interface{}{ + "container.log", + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + FieldReplaceValues("**.paths"), + FieldAppendValues("**.processors"), + }, + func(t *testing.T, c *Config) { + unpacked := make(map[string]interface{}) + assert.NoError(t, c.Unpack(unpacked)) + + paths, _ := unpacked["paths"] + assert.Len(t, paths, 1) + assert.Equal(t, []interface{}{"container.log"}, paths) + + processors, _ := unpacked["processors"] + assert.Len(t, processors, 2) + + processorNames := make([]string, 2) + procs := processors.([]interface{}) + for i, proc := range procs { + for name := range proc.(map[string]interface{}) { + processorNames[i] = name + } + } + assert.Equal(t, []string{"add_locale", "add_fields"}, processorNames) + }, + }, + { + "adjust merging based on indexes", + []interface{}{ + map[string]interface{}{ + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "foo": "bar", + }, + }, + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []string{"merged"}, + }, + }, + }, + }, + map[string]interface{}{ + "processors": []interface{}{ + map[string]interface{}{ + "add_locale": map[string]interface{}{}, + }, + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "replace": "no-bar", + }, + }, + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []string{"together"}, + }, + }, + }, + }, + }, + []Option{ + PathSep("."), + FieldReplaceValues("processors.1"), + FieldAppendValues("processors.2.add_tags.tags"), + }, + func(t *testing.T, c *Config) { + unpacked := make(map[string]interface{}) + assert.NoError(t, c.Unpack(unpacked)) + + processors, _ := unpacked["processors"] + assert.Len(t, processors, 3) + + processorsByAction := make(map[string]interface{}) + procs := processors.([]interface{}) + for _, proc := range procs { + for name, val := range proc.(map[string]interface{}) { + processorsByAction[name] = val + } + } + + addFieldsAction, ok := processorsByAction["add_fields"] + assert.True(t, ok) + assert.Equal(t, map[string]interface{}{"replace": "no-bar"}, addFieldsAction) + + addTagsAction, ok := processorsByAction["add_tags"] + assert.True(t, ok) + tags, ok := (addTagsAction.(map[string]interface{}))["tags"] + assert.True(t, ok) + assert.Equal(t, []interface{}{"merged", "together"}, tags) + }, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + c := New() + for _, config := range test.Configs { + assert.NoError(t, c.Merge(config, test.Options...)) + } + test.Assert(t, c) + }) + } +} + func TestMergeSquash(t *testing.T) { type SubType struct{ B bool } type SubInterface struct{ B interface{} } diff --git a/opts.go b/opts.go index ba95424..68c23e5 100644 --- a/opts.go +++ b/opts.go @@ -18,7 +18,9 @@ package ucfg import ( + "fmt" "os" + "strings" "github.com/elastic/go-ucfg/parse" ) @@ -38,6 +40,7 @@ type options struct { noParse bool configValueHandling configHandling + fieldHandlingTree *fieldHandlingTree // temporary cache of parsed splice values for lifetime of call to // Unpack/Pack/Get/... @@ -48,6 +51,9 @@ type options struct { type valueCache map[string]spliceValue +// specific API on top of Config to handle adjusting merging behavior per fields +type fieldHandlingTree Config + // id used to store intermediate parse results in current execution context. // As parsing results might differ between multiple calls due to: // splice being shared between multiple configurations, or environment @@ -162,6 +168,57 @@ func makeOptValueHandling(h configHandling) Option { } } +var ( + // FieldMergeValues option configures all merging and unpacking operations to use + // the default merging behavior for the specified field. This overrides the any struct + // tags during unpack for the field. Nested field names can be defined using dot + // notation. + FieldMergeValues = makeFieldOptValueHandling(cfgMergeValues) + + // FieldReplaceValues option configures all merging and unpacking operations to + // replace old dictionaries and arrays while merging for the specified field. This + // overrides the any struct tags during unpack for the field. Nested field names + // can be defined using dot notation. + FieldReplaceValues = makeFieldOptValueHandling(cfgReplaceValue) + + // FieldAppendValues option configures all merging and unpacking operations to + // merge dictionaries and append arrays to existing arrays while merging for the + // specified field. This overrides the any struct tags during unpack for the field. + // Nested field names can be defined using dot notation. + FieldAppendValues = makeFieldOptValueHandling(cfgArrAppend) + + // FieldPrependValues option configures all merging and unpacking operations to + // merge dictionaries and prepend arrays to existing arrays while merging for the + // specified field. This overrides the any struct tags during unpack for the field. + // Nested field names can be defined using dot notation. + FieldPrependValues = makeFieldOptValueHandling(cfgArrPrepend) +) + +func makeFieldOptValueHandling(h configHandling) func(...string) Option { + return func(fieldName ...string) Option { + if len(fieldName) == 0 { + return func(_ *options) {} + } + + table := make(map[string]configHandling) + for _, name := range fieldName { + // field value config options are rendered into a Config; the '*' represents the handling method + // for everything nested under this field. + if !strings.HasSuffix(name, ".*") { + name = fmt.Sprintf("%s.*", name) + } + table[name] = h + } + + return func(o *options) { + if o.fieldHandlingTree == nil { + o.fieldHandlingTree = newFieldHandlingTree() + } + o.fieldHandlingTree.merge(table, PathSep(o.pathSep)) + } + } +} + // VarExp option enables support for variable expansion. Resolve and Env options will only be effective if VarExp is set. var VarExp Option = doVarExp @@ -200,3 +257,59 @@ func (cache valueCache) cachedValue( } return v, err } + +func newFieldHandlingTree() *fieldHandlingTree { + return (*fieldHandlingTree)(New()) +} + +func (t *fieldHandlingTree) merge(other interface{}, opts ...Option) error { + cfg := (*Config)(t) + return cfg.Merge(other, opts...) +} + +func (t *fieldHandlingTree) child(fieldName string, idx int) (*fieldHandlingTree, error) { + cfg := (*Config)(t) + child, err := cfg.Child(fieldName, idx) + if err != nil { + return nil, err + } + return (*fieldHandlingTree)(child), nil +} + +func (t *fieldHandlingTree) configHandling(fieldName string, idx int) (configHandling, error) { + cfg := (*Config)(t) + handling, err := cfg.Uint(fieldName, idx) + if err != nil { + return cfgDefaultHandling, err + } + return configHandling(handling), nil +} + +func (t *fieldHandlingTree) wildcard() (*fieldHandlingTree, error) { + return t.child("**", -1) +} + +func (t *fieldHandlingTree) setWildcard(wildcard *fieldHandlingTree) error { + cfg := (*Config)(t) + return cfg.SetChild("**", -1, (*Config)(wildcard)) +} + +func (t *fieldHandlingTree) fieldHandling(fieldName string, idx int) (configHandling, *fieldHandlingTree, bool) { + child, err := t.child(fieldName, idx) + if err == nil { + cfgHandling, err := child.configHandling("*", -1) + if err == nil { + return cfgHandling, child, true + } + } + // try wildcard match + wildcard, err := t.wildcard() + if err != nil { + return cfgDefaultHandling, child, false + } + cfgHandling, cfg, ok := wildcard.fieldHandling(fieldName, idx) + if ok { + return cfgHandling, cfg, ok + } + return cfgDefaultHandling, child, ok +}