From cb55dc398a924b0470d4f7ab9e20a03e880e0ca5 Mon Sep 17 00:00:00 2001 From: simitt Date: Sat, 5 Sep 2020 18:40:08 +0200 Subject: [PATCH 1/6] Add metadata modeldecoder and generator logic * Introduce modeldecoder models for metadata with nullable values * Generate model decoding and validation functions * Run generator logic on `make update` --- NOTICE.txt | 2 +- decoder/decoder.go | 45 ++ go.mod | 4 +- go.sum | 2 + main.go | 1 + model/modeldecoder/cmd/main.go | 86 +++ model/modeldecoder/generator/generator.go | 586 +++++++++++++++++ model/modeldecoder/rumv3/decoder.go | 111 ++++ model/modeldecoder/rumv3/model.go | 82 +++ model/modeldecoder/rumv3/model_generated.go | 277 ++++++++ model/modeldecoder/typ/typ.go | 147 +++++ model/modeldecoder/typ/typ_test.go | 156 +++++ model/modeldecoder/v2/decoder.go | 231 +++++++ model/modeldecoder/v2/decoder_test.go | 169 +++++ model/modeldecoder/v2/model.go | 158 +++++ model/modeldecoder/v2/model_generated.go | 610 ++++++++++++++++++ model/modeldecoder/v2/model_test.go | 341 ++++++++++ .../package_tests/metadata_attrs_test.go | 6 +- testdata/intake-v2/metadata.ndjson | 1 + testdata/intake-v2/only-metadata.ndjson | 1 - testdata/intake-v3/metadata.ndjson | 1 + tests/system/test_requests.py | 2 +- 22 files changed, 3011 insertions(+), 8 deletions(-) create mode 100644 decoder/decoder.go create mode 100644 model/modeldecoder/cmd/main.go create mode 100644 model/modeldecoder/generator/generator.go create mode 100644 model/modeldecoder/rumv3/decoder.go create mode 100644 model/modeldecoder/rumv3/model.go create mode 100644 model/modeldecoder/rumv3/model_generated.go create mode 100644 model/modeldecoder/typ/typ.go create mode 100644 model/modeldecoder/typ/typ_test.go create mode 100644 model/modeldecoder/v2/decoder.go create mode 100644 model/modeldecoder/v2/decoder_test.go create mode 100644 model/modeldecoder/v2/model.go create mode 100644 model/modeldecoder/v2/model_generated.go create mode 100644 model/modeldecoder/v2/model_test.go create mode 100644 testdata/intake-v2/metadata.ndjson delete mode 100644 testdata/intake-v2/only-metadata.ndjson create mode 100644 testdata/intake-v3/metadata.ndjson diff --git a/NOTICE.txt b/NOTICE.txt index d10bc37cbc2..4003a8a216f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2839,7 +2839,7 @@ Contents of "LICENSE": -------------------------------------------------------------------- Dependency: github.com/json-iterator/go -Version: v1.1.8 +Version: v1.1.10 License type (autodetected): MIT Contents of "LICENSE": diff --git a/decoder/decoder.go b/decoder/decoder.go new file mode 100644 index 00000000000..439bd90c2f6 --- /dev/null +++ b/decoder/decoder.go @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decoder + +import ( + "io" + + jsoniter "github.com/json-iterator/go" +) + +//TODO(simitt): look into config options for performance tuning +var jsonit = jsoniter.ConfigCompatibleWithStandardLibrary + +type Decoder interface { + Decode(v interface{}) error +} + +// JSONIterDecoder can decode from a given reader, using jsoniter +// TODO(simitt): rename to JSONDecoder when everything is integrated +type JSONIterDecoder struct { + *jsoniter.Decoder +} + +// NewJSONIteratorDecoder returns a *json.Decoder where numbers are unmarshaled +// as a Number instead of a float64 into an interface{} +func NewJSONIteratorDecoder(r io.Reader) JSONIterDecoder { + d := jsonit.NewDecoder(r) + d.UseNumber() + return JSONIterDecoder{Decoder: d} +} diff --git a/go.mod b/go.mod index 40c4faec44c..10c4edea61d 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/jaegertracing/jaeger v1.16.0 github.com/jcmturner/gofork v1.0.0 // indirect github.com/josephspurrier/goversioninfo v1.2.0 // indirect - github.com/json-iterator/go v1.1.8 + github.com/json-iterator/go v1.1.10 github.com/jstemmer/go-junit-report v0.9.1 github.com/klauspost/compress v1.9.3-0.20191122130757-c099ac9f21dd // indirect github.com/kr/pretty v0.2.0 // indirect @@ -75,7 +75,7 @@ require ( golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 - golang.org/x/tools v0.0.0-20200823205832-c024452afbcd // indirect + golang.org/x/tools v0.0.0-20200823205832-c024452afbcd google.golang.org/grpc v1.29.1 gopkg.in/yaml.v2 v2.3.0 howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5 // indirect diff --git a/go.sum b/go.sum index f2e559a2de0..c82d234375c 100644 --- a/go.sum +++ b/go.sum @@ -642,6 +642,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= diff --git a/main.go b/main.go index 9bdaf0ec8fb..82d5b49a94b 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ package main //go:generate go run script/inline_schemas/inline_schemas.go +//go:generate go run model/modeldecoder/cmd/main.go import ( "os" diff --git a/model/modeldecoder/cmd/main.go b/model/modeldecoder/cmd/main.go new file mode 100644 index 00000000000..b1f081f305a --- /dev/null +++ b/model/modeldecoder/cmd/main.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "bytes" + "go/format" + "os" + "path/filepath" + + "github.com/elastic/apm-server/model/modeldecoder/generator" +) + +const ( + basePath = "github.com/elastic/apm-server" + modeldecoderPath = "model/modeldecoder" +) + +var ( + importPath = filepath.Join(basePath, modeldecoderPath) + typPath = filepath.Join(importPath, "typ") +) + +func main() { + genV2Models() + genRUMV3Models() +} + +func genV2Models() { + pkg := "v2" + rootObjs := []string{"metadataNoKey", "metadataWithKey"} + out := filepath.Join(modeldecoderPath, pkg, "model_generated.go") + gen, err := generator.NewGenerator(importPath, pkg, typPath, rootObjs) + if err != nil { + panic(err) + } + generate(gen, out) +} + +func genRUMV3Models() { + pkg := "rumv3" + rootObjs := []string{"metadataWithKey"} + out := filepath.Join(modeldecoderPath, pkg, "model_generated.go") + gen, err := generator.NewGenerator(importPath, pkg, typPath, rootObjs) + if err != nil { + panic(err) + } + generate(gen, out) +} + +type gen interface { + Generate() (bytes.Buffer, error) +} + +func generate(g gen, p string) { + b, err := g.Generate() + if err != nil { + panic(err) + } + fmtd, err := format.Source(b.Bytes()) + if err != nil { + panic(err) + } + f, err := os.Create(p) + if err != nil { + panic(err) + } + if _, err := f.Write(fmtd); err != nil { + panic(err) + } +} diff --git a/model/modeldecoder/generator/generator.go b/model/modeldecoder/generator/generator.go new file mode 100644 index 00000000000..75b72bce16f --- /dev/null +++ b/model/modeldecoder/generator/generator.go @@ -0,0 +1,586 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package generator + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/token" + "go/types" + "path/filepath" + "reflect" + "sort" + "strings" + + "golang.org/x/tools/go/packages" +) + +// Generator creates following struct methods +// `IsSet() bool` +// `Reset()` +// `validate() error` +// on all structs (exported and unexported) that are referenced +// by at least one of the root types +type Generator struct { + buf bytes.Buffer + pkgName string + rootObjs map[string]structType + // parsed structs from loading types from the provided package + structTypes structTypes + // keep track of already processed types in case one type is + // referenced multiple times + processedTypes map[string]interface{} + + typString, typInt, typInterface string +} + +// NewGenerator takes an importPath and the package name for which +// the type definitions should be loaded. +// The typPkg is used to implement validation rules specific to types +// of the package. The generator creates methods only for types referenced +// directly or indirectly by any of the root types. +func NewGenerator(importPath string, pkg string, typPath string, + root []string) (*Generator, error) { + loaded, err := loadPackage(fmt.Sprintf("%s/%s", importPath, pkg)) + if err != nil { + return nil, err + } + structTypes, err := parseStructTypes(loaded) + if err != nil { + return nil, err + } + g := Generator{ + pkgName: loaded.Types.Name(), + structTypes: structTypes, + rootObjs: make(map[string]structType, len(root)), + processedTypes: make(map[string]interface{}), + typString: fmt.Sprintf("%s.String", typPath), + typInt: fmt.Sprintf("%s.Int", typPath), + typInterface: fmt.Sprintf("%s.Interface", typPath), + } + for _, r := range root { + rootObjPath := fmt.Sprintf("%s.%s", filepath.Join(importPath, pkg), r) + rootObj, ok := structTypes[rootObjPath] + if !ok { + return nil, fmt.Errorf("object with root key %s not found", rootObjPath) + } + g.rootObjs[rootObj.name] = rootObj + } + return &g, nil +} + +// Generate writes generated methods to the buffer +func (g *Generator) Generate() (bytes.Buffer, error) { + fmt.Fprintf(&g.buf, ` +package %s + +import ( + "encoding/json" + "fmt" + "unicode/utf8" +) +`[1:], g.pkgName) + + for _, rootObj := range g.rootObjs { + if err := g.generate(rootObj, ""); err != nil { + return g.buf, err + } + } + return g.buf, nil +} + +const ( + ruleRequired = "required" + ruleMax = "max" + ruleMaxVals = "maxVals" + rulePattern = "pattern" + rulePatternKeys = "patternKeys" + ruleTypes = "types" + ruleTypesVals = "typesVals" +) + +type structTypes map[string]structType + +type structType struct { + name string + fields []structField +} +type structField struct { + name string + typ types.Type + tag reflect.StructTag +} + +// create flattened field keys by recursively iterating through the struct types; +// there is only struct local knowledge and no knowledge about the parent, +// deriving the absolute key is not possible in scenarios where one struct +// type is referenced as a field in multiple struct types +func (g *Generator) generate(st structType, key string) error { + if _, ok := g.processedTypes[st.name]; ok { + return nil + } + g.processedTypes[st.name] = nil + if err := g.generateIsSet(st, key); err != nil { + return err + } + if err := g.generateReset(st, key); err != nil { + return err + } + if err := g.generateValidation(st, key); err != nil { + return err + } + if key != "" { + key += "." + } + for _, f := range st.fields { + if child, ok := g.structTypes[f.typ.String()]; ok { + if err := g.generate(child, fmt.Sprintf("%s%s", key, jsonName(f))); err != nil { + return err + } + } + } + return nil +} + +func (g *Generator) generateIsSet(structTyp structType, key string) error { + fmt.Fprintf(&g.buf, ` +func (m *%s) IsSet() bool { + return`, structTyp.name) + if key != "" { + key += "." + } + for i := 0; i < len(structTyp.fields); i++ { + prefix := ` ||` + if i == 0 { + prefix = `` + } + f := structTyp.fields[i] + + switch t := f.typ.Underlying().(type) { + case *types.Slice, *types.Map: + fmt.Fprintf(&g.buf, `%s len(m.%s) > 0`, prefix, f.name) + case *types.Struct: + fmt.Fprintf(&g.buf, `%s m.%s.IsSet()`, prefix, f.name) + default: + return fmt.Errorf("unhandled type %T for IsSet() for '%s%s'", t, key, jsonName(f)) + } + } + fmt.Fprint(&g.buf, ` +} +`) + return nil +} + +func (g *Generator) generateReset(structTyp structType, key string) error { + fmt.Fprintf(&g.buf, ` +func (m *%s) Reset() { +`, structTyp.name) + if key != "" { + key += "." + } + for _, f := range structTyp.fields { + switch t := f.typ.Underlying().(type) { + case *types.Slice: + // the slice len is set to zero, not returning the underlying + // memory to the garbage collector; when the size of slices differs + // this potentially leads to keeping more memory allocated than required; + // at the moment metadata.process.argv is the only slice + fmt.Fprintf(&g.buf, ` +m.%s = m.%s[:0] +`[1:], f.name, f.name) + case *types.Map: + // the map is cleared, not returning the underlying memory to + // the garbage collector; when map size differs this potentially + // leads to keeping more memory allocated than required + fmt.Fprintf(&g.buf, ` +for k := range m.%s { + delete(m.%s, k) +} +`[1:], f.name, f.name) + case *types.Struct: + fmt.Fprintf(&g.buf, ` +m.%s.Reset() +`[1:], f.name) + default: + return fmt.Errorf("unhandled type %T for Reset() for '%s%s'", t, key, jsonName(f)) + } + } + fmt.Fprint(&g.buf, ` +} +`[1:]) + return nil +} + +func (g *Generator) generateValidation(structTyp structType, key string) error { + fmt.Fprintf(&g.buf, ` +func (m *%s) validate() error { +`, structTyp.name) + if _, ok := g.rootObjs[structTyp.name]; !ok { + fmt.Fprint(&g.buf, ` +if !m.IsSet() { + return nil +} +`[1:]) + } + + if key != "" { + key += "." + } + for i := 0; i < len(structTyp.fields); i++ { + f := structTyp.fields[i] + flattenedName := fmt.Sprintf("%s%s", key, jsonName(f)) + + // if field is a model struct, call its validation function + if _, ok := g.structTypes[f.typ.String()]; ok { + fmt.Fprintf(&g.buf, ` +if err := m.%s.validate(); err != nil{ + return err +} +`[1:], f.name) + } + + parts, err := validationTag(f.tag) + if err != nil { + return fmt.Errorf("'%s': %w", flattenedName, err) + } + // use a sorted slice of tag keys to create tag related + // validation methods in the same output order on every run + var sortedRules = make([]string, 0, len(parts)) + for k := range parts { + sortedRules = append(sortedRules, k) + } + sort.Slice(sortedRules, func(i, j int) bool { + return sortedRules[i] < sortedRules[j] + }) + + switch t := f.typ.Underlying().(type) { + case *types.Slice: + for _, rule := range sortedRules { + switch rule { + case ruleRequired: + fmt.Fprintf(&g.buf, ` +if len(m.%s) == 0{ + return fmt.Errorf("'%s' required") +} +`[1:], f.name, flattenedName) + default: + return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) + } + } + case *types.Map: + var required bool + if _, ok := parts[ruleRequired]; ok { + required = true + delete(parts, ruleRequired) + fmt.Fprintf(&g.buf, ` +if len(m.%s) == 0{ + return fmt.Errorf("'%s' required") +} +`[1:], f.name, flattenedName) + } + if len(parts) == 0 { + continue + } + // iterate over map once and run checks + fmt.Fprintf(&g.buf, ` +for k,v := range m.%s{ +`[1:], f.name) + if regex, ok := parts[rulePatternKeys]; ok { + delete(parts, rulePatternKeys) + fmt.Fprintf(&g.buf, ` +if !%s.MatchString(k){ + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], regex, rulePatternKeys, regex, flattenedName) + } + if types, ok := parts[ruleTypesVals]; ok { + delete(parts, ruleTypesVals) + fmt.Fprintf(&g.buf, ` +switch t := v.(type){ +`[1:]) + if !required { + fmt.Fprintf(&g.buf, ` +case nil: +`[1:]) + } + for _, typ := range strings.Split(types, ";") { + if typ == "number" { + typ = "json.Number" + } + fmt.Fprintf(&g.buf, ` +case %s: +`[1:], typ) + if typ == "string" { + if maxVal, ok := parts[ruleMaxVals]; ok { + delete(parts, ruleMaxVals) + fmt.Fprintf(&g.buf, ` +if utf8.RuneCountInString(t) > %s{ + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], maxVal, ruleMaxVals, maxVal, flattenedName) + } + } + } + fmt.Fprintf(&g.buf, ` +default: + return fmt.Errorf("validation rule '%s(%s)' violated for '%s' for key " + k) +} +`[1:], ruleTypesVals, types, flattenedName) + } + // close iteration over map + fmt.Fprintf(&g.buf, ` +} +`[1:]) + if len(parts) > 0 { + return fmt.Errorf("unhandled tag rule(s) '%v' for '%s'", parts, flattenedName) + } + case *types.Struct: + switch f.typ.String() { + //TODO(simitt): can these type checks be more generic? + case g.typString: + for _, rule := range sortedRules { + val := parts[rule] + switch rule { + case ruleRequired: + fmt.Fprintf(&g.buf, ` +if !m.%s.IsSet() || m.%s.IsNil() { + return fmt.Errorf("'%s' required") +} +`[1:], f.name, f.name, flattenedName) + case ruleMax: + fmt.Fprintf(&g.buf, ` +if utf8.RuneCountInString(m.%s.Val) > %s{ +return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], f.name, val, rule, val, flattenedName) + case rulePattern: + fmt.Fprintf(&g.buf, ` +if !%s.MatchString(m.%s.Val){ + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], val, f.name, rule, val, flattenedName) + default: + return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) + } + } + case g.typInt: + for _, rule := range sortedRules { + val := parts[rule] + switch rule { + case ruleRequired: + fmt.Fprintf(&g.buf, ` +if !m.%s.IsSet() || m.%s.IsNil(){ + return fmt.Errorf("'%s' required") +} +`[1:], f.name, f.name, flattenedName) + case ruleMax: + fmt.Fprintf(&g.buf, ` +if m.%s.Val > %s{ + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], f.name, val, rule, val, flattenedName) + default: + return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) + } + } + case g.typInterface: + var required bool + if _, ok := parts[ruleRequired]; ok { + required = true + } + for _, rule := range sortedRules { + val := parts[rule] + switch rule { + case ruleRequired: + fmt.Fprintf(&g.buf, ` +if !m.%s.IsSet() || m.%s.IsNil(){ + return fmt.Errorf("'%s' required") +} +`[1:], f.name, f.name, flattenedName) + case ruleMax: + //handled in switch statement for string types + case ruleTypes: + fmt.Fprintf(&g.buf, ` +switch t := m.%s.Val.(type){ +`[1:], f.name) + for _, typ := range strings.Split(val, ";") { + if typ == "int" { + fmt.Fprintf(&g.buf, ` +case json.Number: +`[1:]) + fmt.Fprintf(&g.buf, ` +if _, err := t.Int64(); err != nil{ + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], rule, val, flattenedName) + } + fmt.Fprintf(&g.buf, ` +case %s: +`[1:], typ) + if typ == "string" { + if max, ok := parts[ruleMax]; ok { + fmt.Fprintf(&g.buf, ` +if utf8.RuneCountInString(t) > %s{ + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], max, ruleMax, max, flattenedName) + } + } + } + if !required { + fmt.Fprintf(&g.buf, ` +case nil: +`[1:]) + } + fmt.Fprintf(&g.buf, ` +default: + return fmt.Errorf("validation rule '%s(%s)' violated for '%s'") +} +`[1:], rule, val, flattenedName) + default: + return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) + } + } + default: + for _, rule := range sortedRules { + switch rule { + case ruleRequired: + fmt.Fprintf(&g.buf, ` +if !m.%s.IsSet(){ + return fmt.Errorf("'%s' required") +} +`[1:], f.name, flattenedName) + default: + return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) + } + } + } + default: + return fmt.Errorf("unhandled type %T for '%s'", t, flattenedName) + } + } + fmt.Fprint(&g.buf, ` + return nil +} +`[1:]) + return nil +} +func jsonName(f structField) string { + parts := parseTag(f.tag, "json") + if len(parts) == 0 { + return strings.ToLower(f.name) + } + return parts[0] +} + +func loadPackage(pkg string) (*packages.Package, error) { + cfg := packages.Config{ + Mode: packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo} + pkgs, err := packages.Load(&cfg, pkg) + if err != nil { + return nil, err + } + if packages.PrintErrors(pkgs) > 0 { + return nil, errors.New("packages load error") + } + return pkgs[0], nil +} + +func parseStructTypes(pkg *packages.Package) (structTypes, error) { + structs := make(structTypes) + for _, syntax := range pkg.Syntax { + for _, decl := range syntax.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + obj := pkg.TypesInfo.Defs[typeSpec.Name] + if obj == nil { + continue + } + named := obj.(*types.TypeName).Type().(*types.Named) + typesStruct, ok := named.Underlying().(*types.Struct) + if !ok { + return nil, fmt.Errorf("unhandled type %T", named.Underlying()) + } + numFields := typesStruct.NumFields() + structFields := make([]structField, 0, numFields) + for i := 0; i < numFields; i++ { + f := typesStruct.Field(i) + if !f.Exported() { + continue + } + structFields = append(structFields, structField{ + name: f.Name(), + typ: f.Type(), + tag: reflect.StructTag(typesStruct.Tag(i)), + }) + } + structs[obj.Type().String()] = structType{name: obj.Name(), fields: structFields} + } + } + } + return structs, nil +} + +func parseTag(structTag reflect.StructTag, tagName string) []string { + tag, ok := structTag.Lookup(tagName) + if !ok || tag == "-" { + return nil + } + return strings.Split(tag, ",") +} + +func validationTag(structTag reflect.StructTag) (map[string]string, error) { + parts := parseTag(structTag, "validate") + m := make(map[string]string, len(parts)) + for _, rule := range parts { + parts := strings.Split(rule, "=") + switch len(parts) { + case 1: + // valueless rule e.g. required + if rule != parts[0] { + return nil, fmt.Errorf("malformed tag '%s'", rule) + } + switch rule { + case ruleRequired: + m[rule] = "" + default: + return nil, fmt.Errorf("unhandled tag rule '%s'", rule) + } + case 2: + // rule=value + m[parts[0]] = parts[1] + switch parts[0] { + case ruleMax, ruleMaxVals, rulePattern, rulePatternKeys, ruleTypes, ruleTypesVals: + default: + return nil, fmt.Errorf("unhandled tag rule '%s'", parts[0]) + } + default: + return nil, fmt.Errorf("malformed tag '%s'", rule) + } + } + return m, nil +} diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go new file mode 100644 index 00000000000..9c6061acc8e --- /dev/null +++ b/model/modeldecoder/rumv3/decoder.go @@ -0,0 +1,111 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rumv3 + +import ( + "fmt" + "sync" + + "github.com/elastic/apm-server/decoder" + "github.com/elastic/apm-server/model" + "github.com/elastic/beats/v7/libbeat/common" +) + +func init() { + metadataPool.New = func() interface{} { + return &metadata{} + } +} + +var metadataPool sync.Pool + +func fetchMetadata() *metadata { + return metadataPool.Get().(*metadata) +} +func releaseMetadata(m *metadata) { + m.Reset() + metadataPool.Put(m) +} + +// DecodeMetadata uses the given decoder to create the input models, +// then runs the defined validations on the input models +// and finally maps the values fom the input model to the given *model.Metadata instance +func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { + m := metadataWithKey{Metadata: *fetchMetadata()} + defer releaseMetadata(&m.Metadata) + if err := d.Decode(&m); err != nil { + return err + } + if err := m.validate(); err != nil { + return err + } + mapToModel(&m.Metadata, out) + return nil +} + +func mapToModel(m *metadata, out *model.Metadata) { + // Labels + out.Labels = common.MapStr{} + out.Labels.Update(m.Labels) + + // Service + if !m.Service.Agent.Name.IsNil() { + out.Service.Agent.Name = m.Service.Agent.Name.Val + } + if !m.Service.Agent.Version.IsNil() { + out.Service.Agent.Version = m.Service.Agent.Version.Val + } + if !m.Service.Environment.IsNil() { + out.Service.Environment = m.Service.Environment.Val + } + if !m.Service.Framework.Name.IsNil() { + out.Service.Framework.Name = m.Service.Framework.Name.Val + } + if !m.Service.Framework.Version.IsNil() { + out.Service.Framework.Version = m.Service.Framework.Version.Val + } + if !m.Service.Language.Name.IsNil() { + out.Service.Language.Name = m.Service.Language.Name.Val + } + if !m.Service.Language.Version.IsNil() { + out.Service.Language.Version = m.Service.Language.Version.Val + } + if !m.Service.Name.IsNil() { + out.Service.Name = m.Service.Name.Val + } + if !m.Service.Runtime.Name.IsNil() { + out.Service.Runtime.Name = m.Service.Runtime.Name.Val + } + if !m.Service.Runtime.Version.IsNil() { + out.Service.Runtime.Version = m.Service.Runtime.Version.Val + } + if !m.Service.Version.IsNil() { + out.Service.Version = m.Service.Version.Val + } + + // User + if !m.User.ID.IsNil() { + out.User.ID = fmt.Sprint(m.User.ID.Val) + } + if !m.User.Email.IsNil() { + out.User.Email = m.User.Email.Val + } + if !m.User.Name.IsNil() { + out.User.Name = m.User.Name.Val + } +} diff --git a/model/modeldecoder/rumv3/model.go b/model/modeldecoder/rumv3/model.go new file mode 100644 index 00000000000..5b9254e4709 --- /dev/null +++ b/model/modeldecoder/rumv3/model.go @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rumv3 + +import ( + "regexp" + + "github.com/elastic/apm-server/model/modeldecoder/typ" + "github.com/elastic/beats/v7/libbeat/common" +) + +var ( + alphaNumericExtRegex = regexp.MustCompile("^[a-zA-Z0-9 _-]+$") + labelsRegex = regexp.MustCompile("^[^.*\"]*$") //do not allow '.' '*' '"' +) + +// metadata contain event metadata +type metadata struct { + Labels common.MapStr `json:"l" validate:"patternKeys=labelsRegex,typesVals=string;bool;number,maxVals=1024"` + Service metadataService `json:"se" validate:"required"` + User metadataUser `json:"u"` +} + +// metadataService holds information about where the data was collected +type metadataService struct { + Agent metadataServiceAgent `json:"a" validate:"required"` + Environment typ.String `json:"en" validate:"max=1024"` + Framework MetadataServiceFramework `json:"fw"` + Language metadataServiceLanguage `json:"la"` + Name typ.String `json:"n" validate:"required,max=1024,pattern=alphaNumericExtRegex"` + Runtime metadataServiceRuntime `json:"ru"` + Version typ.String `json:"ve" validate:"max=1024"` +} + +//metadataServiceAgent has a version and a name +type metadataServiceAgent struct { + Name typ.String `json:"n" validate:"required,max=1024"` + Version typ.String `json:"ve" validate:"required,max=1024"` +} + +//MetadataServiceFramework has a version and name +type MetadataServiceFramework struct { + Name typ.String `json:"n" validate:"max=1024"` + Version typ.String `json:"ve" validate:"max=1024"` +} + +//MetadataLanguage has a version and name +type metadataServiceLanguage struct { + Name typ.String `json:"n" validate:"required,max=1024"` + Version typ.String `json:"ve" validate:"max=1024"` +} + +//MetadataRuntime has a version and name +type metadataServiceRuntime struct { + Name typ.String `json:"n" validate:"required,max=1024"` + Version typ.String `json:"ve" validate:"required,max=1024"` +} + +type metadataUser struct { + ID typ.Interface `json:"id" validate:"max=1024,types=string;int"` + Email typ.String `json:"em" validate:"max=1024"` + Name typ.String `json:"un" validate:"max=1024"` +} + +type metadataWithKey struct { + Metadata metadata `json:"m" validate:"required"` +} diff --git a/model/modeldecoder/rumv3/model_generated.go b/model/modeldecoder/rumv3/model_generated.go new file mode 100644 index 00000000000..942e1f5cb83 --- /dev/null +++ b/model/modeldecoder/rumv3/model_generated.go @@ -0,0 +1,277 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rumv3 + +import ( + "encoding/json" + "fmt" + "unicode/utf8" +) + +func (m *metadataWithKey) IsSet() bool { + return m.Metadata.IsSet() +} + +func (m *metadataWithKey) Reset() { + m.Metadata.Reset() +} + +func (m *metadataWithKey) validate() error { + if err := m.Metadata.validate(); err != nil { + return err + } + if !m.Metadata.IsSet() { + return fmt.Errorf("'m' required") + } + return nil +} + +func (m *metadata) IsSet() bool { + return len(m.Labels) > 0 || m.Service.IsSet() || m.User.IsSet() +} + +func (m *metadata) Reset() { + for k := range m.Labels { + delete(m.Labels, k) + } + m.Service.Reset() + m.User.Reset() +} + +func (m *metadata) validate() error { + if !m.IsSet() { + return nil + } + for k, v := range m.Labels { + if !labelsRegex.MatchString(k) { + return fmt.Errorf("validation rule 'patternKeys(labelsRegex)' violated for 'm.l'") + } + switch t := v.(type) { + case nil: + case string: + if utf8.RuneCountInString(t) > 1024 { + return fmt.Errorf("validation rule 'maxVals(1024)' violated for 'm.l'") + } + case bool: + case json.Number: + default: + return fmt.Errorf("validation rule 'typesVals(string;bool;number)' violated for 'm.l' for key " + k) + } + } + if err := m.Service.validate(); err != nil { + return err + } + if !m.Service.IsSet() { + return fmt.Errorf("'m.se' required") + } + if err := m.User.validate(); err != nil { + return err + } + return nil +} + +func (m *metadataService) IsSet() bool { + return m.Agent.IsSet() || m.Environment.IsSet() || m.Framework.IsSet() || m.Language.IsSet() || m.Name.IsSet() || m.Runtime.IsSet() || m.Version.IsSet() +} + +func (m *metadataService) Reset() { + m.Agent.Reset() + m.Environment.Reset() + m.Framework.Reset() + m.Language.Reset() + m.Name.Reset() + m.Runtime.Reset() + m.Version.Reset() +} + +func (m *metadataService) validate() error { + if !m.IsSet() { + return nil + } + if err := m.Agent.validate(); err != nil { + return err + } + if !m.Agent.IsSet() { + return fmt.Errorf("'m.se.a' required") + } + if utf8.RuneCountInString(m.Environment.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.en'") + } + if err := m.Framework.validate(); err != nil { + return err + } + if err := m.Language.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.n'") + } + if !alphaNumericExtRegex.MatchString(m.Name.Val) { + return fmt.Errorf("validation rule 'pattern(alphaNumericExtRegex)' violated for 'm.se.n'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'m.se.n' required") + } + if err := m.Runtime.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.ve'") + } + return nil +} + +func (m *metadataServiceAgent) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceAgent) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceAgent) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.a.n'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'m.se.a.n' required") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.a.ve'") + } + if !m.Version.IsSet() || m.Version.IsNil() { + return fmt.Errorf("'m.se.a.ve' required") + } + return nil +} + +func (m *MetadataServiceFramework) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *MetadataServiceFramework) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *MetadataServiceFramework) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.fw.n'") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.fw.ve'") + } + return nil +} + +func (m *metadataServiceLanguage) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceLanguage) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceLanguage) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.la.n'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'m.se.la.n' required") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.la.ve'") + } + return nil +} + +func (m *metadataServiceRuntime) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceRuntime) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceRuntime) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.ru.n'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'m.se.ru.n' required") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.ru.ve'") + } + if !m.Version.IsSet() || m.Version.IsNil() { + return fmt.Errorf("'m.se.ru.ve' required") + } + return nil +} + +func (m *metadataUser) IsSet() bool { + return m.ID.IsSet() || m.Email.IsSet() || m.Name.IsSet() +} + +func (m *metadataUser) Reset() { + m.ID.Reset() + m.Email.Reset() + m.Name.Reset() +} + +func (m *metadataUser) validate() error { + if !m.IsSet() { + return nil + } + switch t := m.ID.Val.(type) { + case string: + if utf8.RuneCountInString(t) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.u.id'") + } + case json.Number: + if _, err := t.Int64(); err != nil { + return fmt.Errorf("validation rule 'types(string;int)' violated for 'm.u.id'") + } + case int: + case nil: + default: + return fmt.Errorf("validation rule 'types(string;int)' violated for 'm.u.id'") + } + if utf8.RuneCountInString(m.Email.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.u.em'") + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'm.u.un'") + } + return nil +} diff --git a/model/modeldecoder/typ/typ.go b/model/modeldecoder/typ/typ.go new file mode 100644 index 00000000000..0f9fa3e1c9d --- /dev/null +++ b/model/modeldecoder/typ/typ.go @@ -0,0 +1,147 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package typ + +import ( + "unsafe" + + jsoniter "github.com/json-iterator/go" +) + +func init() { + jsoniter.RegisterTypeDecoderFunc("typ.String", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + switch iter.WhatIsNext() { + case jsoniter.NilValue: + iter.ReadNil() + default: + (*((*String)(ptr))).Val = iter.ReadString() + (*((*String)(ptr))).isNotNil = true + } + (*((*String)(ptr))).isSet = true + }) + jsoniter.RegisterTypeDecoderFunc("typ.Int", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + switch iter.WhatIsNext() { + case jsoniter.NilValue: + iter.ReadNil() + default: + (*((*Int)(ptr))).Val = iter.ReadInt() + (*((*Int)(ptr))).isNotNil = true + } + (*((*Int)(ptr))).isSet = true + }) + jsoniter.RegisterTypeDecoderFunc("typ.Interface", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + switch iter.WhatIsNext() { + case jsoniter.NilValue: + iter.ReadNil() + default: + (*((*Interface)(ptr))).Val = iter.Read() + (*((*Interface)(ptr))).isNotNil = true + } + (*((*Interface)(ptr))).isSet = true + }) +} + +type String struct { + Val string + isSet, isNotNil bool +} + +// Set sets the value +func (v *String) Set(val string) { + v.Val = val + v.isSet = true + v.isNotNil = true +} + +// IsSet is true when decode was called +func (v *String) IsSet() bool { + return v.isSet +} + +// IsNil is true when no value or null value was decoded +func (v *String) IsNil() bool { + return !v.isNotNil +} + +// Reset sets the String to it's initial state +// where it is not set and has no value +func (v *String) Reset() { + v.Val = "" + v.isSet = false + v.isNotNil = false +} + +type Int struct { + Val int + isSet, isNotNil bool +} + +// Set sets the value +func (v *Int) Set(val int) { + v.Val = val + v.isSet = true + v.isNotNil = true +} + +// IsSet is true when decode was called +func (v *Int) IsSet() bool { + return v.isSet +} + +// IsNil is true when no value or null value was decoded +func (v *Int) IsNil() bool { + return !v.isNotNil +} + +// Reset sets the Int to it's initial state +// where it is not set and has no value +func (v *Int) Reset() { + v.Val = 0 + v.isSet = false + v.isNotNil = false +} + +type Interface struct { + Val interface{} `json:"val,omitempty"` + isSet, isNotNil bool +} + +// Set sets the value +func (v *Interface) Set(val interface{}) { + v.Val = val + v.isSet = true + v.isNotNil = true +} + +// IsSet is true when decode was called +func (v *Interface) IsSet() bool { + return v.isSet +} + +// IsNil is true when no value or null value was decoded +func (v *Interface) IsNil() bool { + return !v.isNotNil +} + +// Reset sets the Interface to it's initial state +// where it is not set and has no value +func (v *Interface) Reset() { + v.Val = nil + v.isSet = false + v.isNotNil = false +} diff --git a/model/modeldecoder/typ/typ_test.go b/model/modeldecoder/typ/typ_test.go new file mode 100644 index 00000000000..509c189298c --- /dev/null +++ b/model/modeldecoder/typ/typ_test.go @@ -0,0 +1,156 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package typ + +import ( + "strings" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testType struct { + S String `json:"s"` + I Int `json:"i"` + V Interface `json:"v"` +} + +var json = jsoniter.ConfigCompatibleWithStandardLibrary + +func TestString(t *testing.T) { + for _, tc := range []struct { + name string + input string + + val string + isSet, isNil, fail bool + }{ + {name: "values", input: `{"s":"agent-go"}`, val: "agent-go", isSet: true}, + {name: "empty", input: `{"s":""}`, isSet: true}, + {name: "null", input: `{"s":null}`, isSet: true, isNil: true}, + {name: "missing", input: `{}`, isNil: true}, + {name: "invalid", input: `{"s":1234}`, isNil: true, fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(tc.input)) + var testStruct testType + err := dec.Decode(&testStruct) + if tc.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.isNil, testStruct.S.IsNil()) + assert.Equal(t, tc.isSet, testStruct.S.IsSet()) + assert.Equal(t, tc.val, testStruct.S.Val) + } + + testStruct.S.Reset() + assert.True(t, testStruct.S.IsNil()) + assert.False(t, testStruct.S.IsSet()) + assert.Empty(t, testStruct.S.Val) + + testStruct.S.Set("teststring") + assert.False(t, testStruct.S.IsNil()) + assert.True(t, testStruct.S.IsSet()) + assert.Equal(t, "teststring", testStruct.S.Val) + }) + } +} + +func TestInt(t *testing.T) { + for _, tc := range []struct { + name string + input string + + val int + isSet, isNil, fail bool + }{ + {name: "values", input: `{"i":44}`, val: 44, isSet: true}, + {name: "empty", input: `{"i":0}`, isSet: true}, + {name: "null", input: `{"i":null}`, isSet: true, isNil: true}, + {name: "missing", input: `{}`, isNil: true}, + {name: "invalid", input: `{"i":"1.0.1"}`, isNil: true, fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(tc.input)) + var testStruct testType + err := dec.Decode(&testStruct) + if tc.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.isNil, testStruct.I.IsNil()) + assert.Equal(t, tc.isSet, testStruct.I.IsSet()) + assert.Equal(t, tc.val, testStruct.I.Val) + } + + testStruct.I.Reset() + assert.True(t, testStruct.I.IsNil()) + assert.False(t, testStruct.I.IsSet()) + assert.Empty(t, testStruct.I.Val) + + testStruct.I.Set(55) + assert.False(t, testStruct.I.IsNil()) + assert.True(t, testStruct.I.IsSet()) + assert.Equal(t, 55, testStruct.I.Val) + }) + } +} + +func TestInterface(t *testing.T) { + for _, tc := range []struct { + name string + input string + + val interface{} + isSet, isNil, fail bool + }{ + {name: "integer", input: `{"v":44}`, val: float64(44), isSet: true}, + {name: "string", input: `{"v":"1.0.1"}`, val: "1.0.1", isSet: true}, + {name: "bool", input: `{"v":true}`, val: true, isSet: true}, + {name: "empty", input: `{"v":""}`, val: "", isSet: true}, + {name: "null", input: `{"v":null}`, isSet: true, isNil: true}, + {name: "missing", input: `{}`, isNil: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(tc.input)) + var testStruct testType + err := dec.Decode(&testStruct) + if tc.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.isNil, testStruct.V.IsNil()) + assert.Equal(t, tc.isSet, testStruct.V.IsSet()) + assert.Equal(t, tc.val, testStruct.V.Val) + } + + testStruct.V.Reset() + assert.True(t, testStruct.V.IsNil()) + assert.False(t, testStruct.V.IsSet()) + assert.Empty(t, testStruct.V.Val) + + testStruct.V.Set("teststring") + assert.False(t, testStruct.V.IsNil()) + assert.True(t, testStruct.V.IsSet()) + assert.Equal(t, "teststring", testStruct.V.Val) + }) + } +} diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go new file mode 100644 index 00000000000..fb9e047cdd8 --- /dev/null +++ b/model/modeldecoder/v2/decoder.go @@ -0,0 +1,231 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v2 + +import ( + "fmt" + "net" + "sync" + + "github.com/elastic/apm-server/decoder" + "github.com/elastic/apm-server/model" + "github.com/elastic/beats/v7/libbeat/common" +) + +func init() { + metadataWithKeyPool.New = func() interface{} { + return &metadataWithKey{} + } + metadataNoKeyPool.New = func() interface{} { + return &metadataNoKey{} + } +} + +var metadataWithKeyPool, metadataNoKeyPool sync.Pool + +func fetchMetadataWithKey() *metadataWithKey { + return metadataWithKeyPool.Get().(*metadataWithKey) +} +func releaseMetadataWithKey(m *metadataWithKey) { + m.Reset() + metadataWithKeyPool.Put(m) +} + +func fetchMetadataNoKey() *metadataNoKey { + return metadataNoKeyPool.Get().(*metadataNoKey) +} +func releaseMetadataNoKey(m *metadataNoKey) { + m.Reset() + metadataNoKeyPool.Put(m) +} + +// DecodeProfileMetadata uses the given decoder to create the input models, +// then runs the defined validations on the input models +// and finally maps the values fom the input model to the given *model.Metadata instance +func DecodeProfileMetadata(d decoder.Decoder, out *model.Metadata) error { + var err error + m := fetchMetadataNoKey() + defer releaseMetadataNoKey(m) + if err = d.Decode(&m.Metadata); err != nil { + return fmt.Errorf("decode error %w", err) + } + if err := m.validate(); err != nil { + return fmt.Errorf("validation error %w", err) + } + mapToMetadataModel(&m.Metadata, out) + return nil +} + +// DecodeMetadata uses the given decoder to create the input models, +// then runs the defined validations on the input models +// and finally maps the values fom the input model to the given *model.Metadata instance +func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { + var err error + m := fetchMetadataWithKey() + defer releaseMetadataWithKey(m) + if err = d.Decode(m); err != nil { + return fmt.Errorf("decode error %w", err) + } + if err := m.validate(); err != nil { + return fmt.Errorf("validation error %w", err) + } + mapToMetadataModel(&m.Metadata, out) + return nil +} + +func mapToMetadataModel(m *metadata, out *model.Metadata) { + // Cloud + if !m.Cloud.Account.ID.IsNil() { + out.Cloud.AccountID = m.Cloud.Account.ID.Val + } + if !m.Cloud.Account.Name.IsNil() { + out.Cloud.AccountName = m.Cloud.Account.Name.Val + } + if !m.Cloud.AvailabilityZone.IsNil() { + out.Cloud.AvailabilityZone = m.Cloud.AvailabilityZone.Val + } + if !m.Cloud.Instance.ID.IsNil() { + out.Cloud.InstanceID = m.Cloud.Instance.ID.Val + } + if !m.Cloud.Instance.Name.IsNil() { + out.Cloud.InstanceName = m.Cloud.Instance.Name.Val + } + if !m.Cloud.Machine.Type.IsNil() { + out.Cloud.MachineType = m.Cloud.Machine.Type.Val + } + if !m.Cloud.Project.ID.IsNil() { + out.Cloud.ProjectID = m.Cloud.Project.ID.Val + } + if !m.Cloud.Project.Name.IsNil() { + out.Cloud.ProjectName = m.Cloud.Project.Name.Val + } + if !m.Cloud.Provider.IsNil() { + out.Cloud.Provider = m.Cloud.Provider.Val + } + if !m.Cloud.Region.IsNil() { + out.Cloud.Region = m.Cloud.Region.Val + } + + // Labels + if len(m.Labels) > 0 { + out.Labels = common.MapStr{} + out.Labels.Update(m.Labels) + } + + // Process + if len(m.Process.Argv) > 0 { + out.Process.Argv = m.Process.Argv + } + if !m.Process.Pid.IsNil() { + out.Process.Pid = m.Process.Pid.Val + } + if !m.Process.Ppid.IsNil() { + var pid = m.Process.Ppid.Val + out.Process.Ppid = &pid + } + if !m.Process.Title.IsNil() { + out.Process.Title = m.Process.Title.Val + } + + // Service + if !m.Service.Agent.EphemeralID.IsNil() { + out.Service.Agent.EphemeralID = m.Service.Agent.EphemeralID.Val + } + if !m.Service.Agent.Name.IsNil() { + out.Service.Agent.Name = m.Service.Agent.Name.Val + } + if !m.Service.Agent.Version.IsNil() { + out.Service.Agent.Version = m.Service.Agent.Version.Val + } + if !m.Service.Environment.IsNil() { + out.Service.Environment = m.Service.Environment.Val + } + if !m.Service.Framework.Name.IsNil() { + out.Service.Framework.Name = m.Service.Framework.Name.Val + } + if !m.Service.Framework.Version.IsNil() { + out.Service.Framework.Version = m.Service.Framework.Version.Val + } + if !m.Service.Language.Name.IsNil() { + out.Service.Language.Name = m.Service.Language.Name.Val + } + if !m.Service.Language.Version.IsNil() { + out.Service.Language.Version = m.Service.Language.Version.Val + } + if !m.Service.Name.IsNil() { + out.Service.Name = m.Service.Name.Val + } + if !m.Service.Node.Name.IsNil() { + out.Service.Node.Name = m.Service.Node.Name.Val + } + if !m.Service.Runtime.Name.IsNil() { + out.Service.Runtime.Name = m.Service.Runtime.Name.Val + } + if !m.Service.Runtime.Version.IsNil() { + out.Service.Runtime.Version = m.Service.Runtime.Version.Val + } + if !m.Service.Version.IsNil() { + out.Service.Version = m.Service.Version.Val + } + + // System + if !m.System.Architecture.IsNil() { + out.System.Architecture = m.System.Architecture.Val + } + if !m.System.ConfiguredHostname.IsNil() { + out.System.ConfiguredHostname = m.System.ConfiguredHostname.Val + } + if !m.System.Container.ID.IsNil() { + out.System.Container.ID = m.System.Container.ID.Val + } + if !m.System.DetectedHostname.IsNil() { + out.System.DetectedHostname = m.System.DetectedHostname.Val + } + if m.System.ConfiguredHostname.IsNil() && m.System.DetectedHostname.IsNil() { + out.System.DetectedHostname = m.System.HostnameDeprecated.Val + } + if !m.System.IP.IsNil() { + out.System.IP = net.ParseIP(m.System.IP.Val) + } + if !m.System.Kubernetes.Namespace.IsNil() { + out.System.Kubernetes.Namespace = m.System.Kubernetes.Namespace.Val + } + if !m.System.Kubernetes.Node.Name.IsNil() { + out.System.Kubernetes.NodeName = m.System.Kubernetes.Node.Name.Val + } + if !m.System.Kubernetes.Pod.Name.IsNil() { + out.System.Kubernetes.PodName = m.System.Kubernetes.Pod.Name.Val + } + if !m.System.Kubernetes.Pod.UID.IsNil() { + out.System.Kubernetes.PodUID = m.System.Kubernetes.Pod.UID.Val + } + if !m.System.Platform.IsNil() { + out.System.Platform = m.System.Platform.Val + } + + // User + if !m.User.ID.IsNil() { + out.User.ID = fmt.Sprint(m.User.ID.Val) + } + if !m.User.Email.IsNil() { + out.User.Email = m.User.Email.Val + } + if !m.User.Name.IsNil() { + out.User.Name = m.User.Name.Val + } +} diff --git a/model/modeldecoder/v2/decoder_test.go b/model/modeldecoder/v2/decoder_test.go new file mode 100644 index 00000000000..677351cbe44 --- /dev/null +++ b/model/modeldecoder/v2/decoder_test.go @@ -0,0 +1,169 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v2 + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/decoder" + "github.com/elastic/apm-server/model" +) + +func TestResetModelOnRelease(t *testing.T) { + inp := `{"metadata":{"service":{"name":"service-a"}}}` + m := fetchMetadataWithKey() + require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(inp)).Decode(m)) + require.True(t, m.IsSet()) + releaseMetadataWithKey(m) + assert.False(t, m.IsSet()) + + mNoKey := fetchMetadataNoKey() + require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(inp)).Decode(mNoKey)) + require.True(t, mNoKey.IsSet()) + releaseMetadataNoKey(mNoKey) + assert.False(t, mNoKey.IsSet()) +} + +func TestDecodeProfileMetadata(t *testing.T) { + var testMinValidMetadata = ` + {"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}` + + decodeProfileMetadata := func(out *model.Metadata) { + dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) + require.NoError(t, DecodeProfileMetadata(dec, out)) + } + + t.Run("fetch-release", func(t *testing.T) { + // this test cannot be run in parallel with other tests using the + // metadataNoKeyPool sync.Pool + + // whenever DecodeProfileMetadata is called a metadata instance + // should be retrieved from the sync.Pool and be released on finish + // test behavior by overriding the new method, occupying an existing + // instance and counting how often the New method is called + var newCount, expectedNewCount int + origNew := metadataNoKeyPool.New + defer func() { metadataNoKeyPool.New = origNew }() + metadataNoKeyPool.New = func() interface{} { + newCount++ + return &metadataNoKey{} + } + var out model.Metadata + // on the first call align the expected with the current new count + // important since other tests might have run already + decodeProfileMetadata(&out) + expectedNewCount = newCount + // on the second call it should reuse the metadata instance + decodeProfileMetadata(&out) + assert.Equal(t, expectedNewCount, newCount) + // force a new instance on the next decoder call + fetchMetadataNoKey() + decodeProfileMetadata(&out) + assert.Equal(t, expectedNewCount+1, newCount) + }) + + t.Run("decode", func(t *testing.T) { + var out model.Metadata + decodeProfileMetadata(&out) + assert.Equal(t, model.Metadata{Service: model.Service{ + Name: "user-service", + Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) + + err := DecodeProfileMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode") + }) + + t.Run("validate", func(t *testing.T) { + inp := `{}` + var out model.Metadata + err := DecodeProfileMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation") + }) + +} + +func TestDecodeMetadata(t *testing.T) { + var testMinValidMetadata = ` + {"metadata":{"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}}` + + decodeMetadata := func(out *model.Metadata) { + dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) + require.NoError(t, DecodeMetadata(dec, out)) + } + + t.Run("fetch-release", func(t *testing.T) { + // this test cannot be run in parallel with other tests using the + // metadataNoKeyPool sync.Pool + + // whenever DecodeProfileMetadata is called a metadata instance + // should be retrieved from the sync.Pool and be released on finish + // test behavior by overriding the new method, occupying an existing + // instance and counting how often the New method is called + var newCount, expectedNewCount int + origNew := metadataWithKeyPool.New + defer func() { metadataWithKeyPool.New = origNew }() + metadataWithKeyPool.New = func() interface{} { + newCount++ + return &metadataWithKey{} + } + var out model.Metadata + // on the first call align the expected with the current new count + // important since other tests might have run already + decodeMetadata(&out) + expectedNewCount = newCount + // on the second call it should reuse the metadata instance + decodeMetadata(&out) + assert.Equal(t, expectedNewCount, newCount) + // force a new instance on the next decoder call + fetchMetadataWithKey() + decodeMetadata(&out) + assert.Equal(t, expectedNewCount+1, newCount) + }) + + t.Run("decode", func(t *testing.T) { + var out model.Metadata + decodeMetadata(&out) + assert.Equal(t, model.Metadata{Service: model.Service{ + Name: "user-service", + Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) + + err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode") + }) + + t.Run("validate", func(t *testing.T) { + inp := `{}` + var out model.Metadata + err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation") + }) + +} + +func TestMappingToModel(t *testing.T) { + //TODO(simitt) + // override existing values if they are set +} diff --git a/model/modeldecoder/v2/model.go b/model/modeldecoder/v2/model.go new file mode 100644 index 00000000000..df8700505f3 --- /dev/null +++ b/model/modeldecoder/v2/model.go @@ -0,0 +1,158 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v2 + +import ( + "regexp" + + "github.com/elastic/apm-server/model/modeldecoder/typ" + "github.com/elastic/beats/v7/libbeat/common" +) + +var ( + alphaNumericExtRegex = regexp.MustCompile("^[a-zA-Z0-9 _-]+$") + labelsRegex = regexp.MustCompile("^[^.*\"]*$") //do not allow '.' '*' '"' +) + +type metadata struct { + Cloud metadataCloud `json:"cloud"` + Labels common.MapStr `json:"labels" validate:"patternKeys=labelsRegex,typesVals=string;bool;number,maxVals=1024"` + Process metadataProcess `json:"process"` + Service metadataService `json:"service" validate:"required"` + System metadataSystem `json:"system"` + User metadataUser `json:"user"` +} + +type metadataCloud struct { + Account metadataCloudAccount `json:"account"` + AvailabilityZone typ.String `json:"availability_zone" validate:"max=1024"` + Instance metadataCloudInstance `json:"instance"` + Machine metadataCloudMachine `json:"machine"` + Project metadataCloudProject `json:"project"` + Provider typ.String `json:"provider" validate:"required,max=1024"` + Region typ.String `json:"region" validate:"max=1024"` +} + +type metadataCloudAccount struct { + ID typ.String `json:"id" validate:"max=1024"` + Name typ.String `json:"name" validate:"max=1024"` +} + +type metadataCloudInstance struct { + ID typ.String `json:"id" validate:"max=1024"` + Name typ.String `json:"name" validate:"max=1024"` +} + +type metadataCloudMachine struct { + Type typ.String `json:"type" validate:"max=1024"` +} + +type metadataCloudProject struct { + ID typ.String `json:"id" validate:"max=1024"` + Name typ.String `json:"name" validate:"max=1024"` +} + +type metadataProcess struct { + Argv []string `json:"argv"` + Pid typ.Int `json:"pid" validate:"required"` + Ppid typ.Int `json:"ppid"` + Title typ.String `json:"title" validate:"max=1024"` +} + +type metadataService struct { + Agent metadataServiceAgent `json:"agent" validate:"required"` + Environment typ.String `json:"environment" validate:"max=1024"` + Framework metadataServiceFramework `json:"framework"` + Language metadataServiceLanguage `json:"language"` + Name typ.String `json:"name" validate:"required,max=1024,pattern=alphaNumericExtRegex"` + Node metadataServiceNode `json:"node"` + Runtime metadataServiceRuntime `json:"runtime"` + Version typ.String `json:"version" validate:"max=1024"` +} + +type metadataServiceAgent struct { + EphemeralID typ.String `json:"ephemeral_id" validate:"max=1024"` + Name typ.String `json:"name" validate:"required,max=1024"` + Version typ.String `json:"version" validate:"required,max=1024"` +} + +type metadataServiceFramework struct { + Name typ.String `json:"name" validate:"max=1024"` + Version typ.String `json:"version" validate:"max=1024"` +} + +type metadataServiceLanguage struct { + Name typ.String `json:"name" validate:"required,max=1024"` + Version typ.String `json:"version" validate:"max=1024"` +} + +type metadataServiceNode struct { + Name typ.String `json:"configured_name" validate:"max=1024"` +} + +type metadataServiceRuntime struct { + Name typ.String `json:"name" validate:"required,max=1024"` + Version typ.String `json:"version" validate:"required,max=1024"` +} + +type metadataSystem struct { + Architecture typ.String `json:"architecture" validate:"max=1024"` + ConfiguredHostname typ.String `json:"configured_hostname" validate:"max=1024"` + Container metadataSystemContainer `json:"container"` + DetectedHostname typ.String `json:"detected_hostname" validate:"max=1024"` + HostnameDeprecated typ.String `json:"hostname" validate:"max=1024"` + IP typ.String `json:"ip"` + Kubernetes metadataSystemKubernetes `json:"kubernetes"` + Platform typ.String `json:"platform" validate:"max=1024"` +} + +type metadataSystemContainer struct { + // `id` is the only field in `system.container`, + // if `system.container:{}` is sent, it should be considered valid + // if additional attributes are defined in the future, add the required tag + ID typ.String `json:"id"` //validate:"required" +} + +type metadataSystemKubernetes struct { + Namespace typ.String `json:"namespace" validate:"max=1024"` + Node metadataSystemKubernetesNode `json:"node"` + Pod metadataSystemKubernetesPod `json:"pod"` +} + +type metadataSystemKubernetesNode struct { + Name typ.String `json:"name" validate:"max=1024"` +} + +type metadataSystemKubernetesPod struct { + Name typ.String `json:"name" validate:"max=1024"` + UID typ.String `json:"uid" validate:"max=1024"` +} + +type metadataUser struct { + ID typ.Interface `json:"id,omitempty" validate:"max=1024,types=string;int"` + Email typ.String `json:"email" validate:"max=1024"` + Name typ.String `json:"username" validate:"max=1024"` +} + +type metadataWithKey struct { + Metadata metadata `json:"metadata" validate:"required"` +} + +type metadataNoKey struct { + Metadata metadata `validate:"required"` +} diff --git a/model/modeldecoder/v2/model_generated.go b/model/modeldecoder/v2/model_generated.go new file mode 100644 index 00000000000..1de9cfe99a1 --- /dev/null +++ b/model/modeldecoder/v2/model_generated.go @@ -0,0 +1,610 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v2 + +import ( + "encoding/json" + "fmt" + "unicode/utf8" +) + +func (m *metadataNoKey) IsSet() bool { + return m.Metadata.IsSet() +} + +func (m *metadataNoKey) Reset() { + m.Metadata.Reset() +} + +func (m *metadataNoKey) validate() error { + if err := m.Metadata.validate(); err != nil { + return err + } + if !m.Metadata.IsSet() { + return fmt.Errorf("'metadata' required") + } + return nil +} + +func (m *metadata) IsSet() bool { + return m.Cloud.IsSet() || len(m.Labels) > 0 || m.Process.IsSet() || m.Service.IsSet() || m.System.IsSet() || m.User.IsSet() +} + +func (m *metadata) Reset() { + m.Cloud.Reset() + for k := range m.Labels { + delete(m.Labels, k) + } + m.Process.Reset() + m.Service.Reset() + m.System.Reset() + m.User.Reset() +} + +func (m *metadata) validate() error { + if !m.IsSet() { + return nil + } + if err := m.Cloud.validate(); err != nil { + return err + } + for k, v := range m.Labels { + if !labelsRegex.MatchString(k) { + return fmt.Errorf("validation rule 'patternKeys(labelsRegex)' violated for 'metadata.labels'") + } + switch t := v.(type) { + case nil: + case string: + if utf8.RuneCountInString(t) > 1024 { + return fmt.Errorf("validation rule 'maxVals(1024)' violated for 'metadata.labels'") + } + case bool: + case json.Number: + default: + return fmt.Errorf("validation rule 'typesVals(string;bool;number)' violated for 'metadata.labels' for key " + k) + } + } + if err := m.Process.validate(); err != nil { + return err + } + if err := m.Service.validate(); err != nil { + return err + } + if !m.Service.IsSet() { + return fmt.Errorf("'metadata.service' required") + } + if err := m.System.validate(); err != nil { + return err + } + if err := m.User.validate(); err != nil { + return err + } + return nil +} + +func (m *metadataCloud) IsSet() bool { + return m.Account.IsSet() || m.AvailabilityZone.IsSet() || m.Instance.IsSet() || m.Machine.IsSet() || m.Project.IsSet() || m.Provider.IsSet() || m.Region.IsSet() +} + +func (m *metadataCloud) Reset() { + m.Account.Reset() + m.AvailabilityZone.Reset() + m.Instance.Reset() + m.Machine.Reset() + m.Project.Reset() + m.Provider.Reset() + m.Region.Reset() +} + +func (m *metadataCloud) validate() error { + if !m.IsSet() { + return nil + } + if err := m.Account.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.AvailabilityZone.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.availability_zone'") + } + if err := m.Instance.validate(); err != nil { + return err + } + if err := m.Machine.validate(); err != nil { + return err + } + if err := m.Project.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.Provider.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.provider'") + } + if !m.Provider.IsSet() || m.Provider.IsNil() { + return fmt.Errorf("'metadata.cloud.provider' required") + } + if utf8.RuneCountInString(m.Region.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.region'") + } + return nil +} + +func (m *metadataCloudAccount) IsSet() bool { + return m.ID.IsSet() || m.Name.IsSet() +} + +func (m *metadataCloudAccount) Reset() { + m.ID.Reset() + m.Name.Reset() +} + +func (m *metadataCloudAccount) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.ID.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.account.id'") + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.account.name'") + } + return nil +} + +func (m *metadataCloudInstance) IsSet() bool { + return m.ID.IsSet() || m.Name.IsSet() +} + +func (m *metadataCloudInstance) Reset() { + m.ID.Reset() + m.Name.Reset() +} + +func (m *metadataCloudInstance) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.ID.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.instance.id'") + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.instance.name'") + } + return nil +} + +func (m *metadataCloudMachine) IsSet() bool { + return m.Type.IsSet() +} + +func (m *metadataCloudMachine) Reset() { + m.Type.Reset() +} + +func (m *metadataCloudMachine) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Type.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.machine.type'") + } + return nil +} + +func (m *metadataCloudProject) IsSet() bool { + return m.ID.IsSet() || m.Name.IsSet() +} + +func (m *metadataCloudProject) Reset() { + m.ID.Reset() + m.Name.Reset() +} + +func (m *metadataCloudProject) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.ID.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.project.id'") + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.project.name'") + } + return nil +} + +func (m *metadataProcess) IsSet() bool { + return len(m.Argv) > 0 || m.Pid.IsSet() || m.Ppid.IsSet() || m.Title.IsSet() +} + +func (m *metadataProcess) Reset() { + m.Argv = m.Argv[:0] + m.Pid.Reset() + m.Ppid.Reset() + m.Title.Reset() +} + +func (m *metadataProcess) validate() error { + if !m.IsSet() { + return nil + } + if !m.Pid.IsSet() || m.Pid.IsNil() { + return fmt.Errorf("'metadata.process.pid' required") + } + if utf8.RuneCountInString(m.Title.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.process.title'") + } + return nil +} + +func (m *metadataService) IsSet() bool { + return m.Agent.IsSet() || m.Environment.IsSet() || m.Framework.IsSet() || m.Language.IsSet() || m.Name.IsSet() || m.Node.IsSet() || m.Runtime.IsSet() || m.Version.IsSet() +} + +func (m *metadataService) Reset() { + m.Agent.Reset() + m.Environment.Reset() + m.Framework.Reset() + m.Language.Reset() + m.Name.Reset() + m.Node.Reset() + m.Runtime.Reset() + m.Version.Reset() +} + +func (m *metadataService) validate() error { + if !m.IsSet() { + return nil + } + if err := m.Agent.validate(); err != nil { + return err + } + if !m.Agent.IsSet() { + return fmt.Errorf("'metadata.service.agent' required") + } + if utf8.RuneCountInString(m.Environment.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.environment'") + } + if err := m.Framework.validate(); err != nil { + return err + } + if err := m.Language.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.name'") + } + if !alphaNumericExtRegex.MatchString(m.Name.Val) { + return fmt.Errorf("validation rule 'pattern(alphaNumericExtRegex)' violated for 'metadata.service.name'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'metadata.service.name' required") + } + if err := m.Node.validate(); err != nil { + return err + } + if err := m.Runtime.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.version'") + } + return nil +} + +func (m *metadataServiceAgent) IsSet() bool { + return m.EphemeralID.IsSet() || m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceAgent) Reset() { + m.EphemeralID.Reset() + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceAgent) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.EphemeralID.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.agent.ephemeral_id'") + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.agent.name'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'metadata.service.agent.name' required") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.agent.version'") + } + if !m.Version.IsSet() || m.Version.IsNil() { + return fmt.Errorf("'metadata.service.agent.version' required") + } + return nil +} + +func (m *metadataServiceFramework) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceFramework) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceFramework) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.framework.name'") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.framework.version'") + } + return nil +} + +func (m *metadataServiceLanguage) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceLanguage) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceLanguage) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.language.name'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'metadata.service.language.name' required") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.language.version'") + } + return nil +} + +func (m *metadataServiceNode) IsSet() bool { + return m.Name.IsSet() +} + +func (m *metadataServiceNode) Reset() { + m.Name.Reset() +} + +func (m *metadataServiceNode) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.node.configured_name'") + } + return nil +} + +func (m *metadataServiceRuntime) IsSet() bool { + return m.Name.IsSet() || m.Version.IsSet() +} + +func (m *metadataServiceRuntime) Reset() { + m.Name.Reset() + m.Version.Reset() +} + +func (m *metadataServiceRuntime) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.runtime.name'") + } + if !m.Name.IsSet() || m.Name.IsNil() { + return fmt.Errorf("'metadata.service.runtime.name' required") + } + if utf8.RuneCountInString(m.Version.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.runtime.version'") + } + if !m.Version.IsSet() || m.Version.IsNil() { + return fmt.Errorf("'metadata.service.runtime.version' required") + } + return nil +} + +func (m *metadataSystem) IsSet() bool { + return m.Architecture.IsSet() || m.ConfiguredHostname.IsSet() || m.Container.IsSet() || m.DetectedHostname.IsSet() || m.HostnameDeprecated.IsSet() || m.IP.IsSet() || m.Kubernetes.IsSet() || m.Platform.IsSet() +} + +func (m *metadataSystem) Reset() { + m.Architecture.Reset() + m.ConfiguredHostname.Reset() + m.Container.Reset() + m.DetectedHostname.Reset() + m.HostnameDeprecated.Reset() + m.IP.Reset() + m.Kubernetes.Reset() + m.Platform.Reset() +} + +func (m *metadataSystem) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Architecture.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.architecture'") + } + if utf8.RuneCountInString(m.ConfiguredHostname.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.configured_hostname'") + } + if err := m.Container.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.DetectedHostname.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.detected_hostname'") + } + if utf8.RuneCountInString(m.HostnameDeprecated.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.hostname'") + } + if err := m.Kubernetes.validate(); err != nil { + return err + } + if utf8.RuneCountInString(m.Platform.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.platform'") + } + return nil +} + +func (m *metadataSystemContainer) IsSet() bool { + return m.ID.IsSet() +} + +func (m *metadataSystemContainer) Reset() { + m.ID.Reset() +} + +func (m *metadataSystemContainer) validate() error { + if !m.IsSet() { + return nil + } + return nil +} + +func (m *metadataSystemKubernetes) IsSet() bool { + return m.Namespace.IsSet() || m.Node.IsSet() || m.Pod.IsSet() +} + +func (m *metadataSystemKubernetes) Reset() { + m.Namespace.Reset() + m.Node.Reset() + m.Pod.Reset() +} + +func (m *metadataSystemKubernetes) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Namespace.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.kubernetes.namespace'") + } + if err := m.Node.validate(); err != nil { + return err + } + if err := m.Pod.validate(); err != nil { + return err + } + return nil +} + +func (m *metadataSystemKubernetesNode) IsSet() bool { + return m.Name.IsSet() +} + +func (m *metadataSystemKubernetesNode) Reset() { + m.Name.Reset() +} + +func (m *metadataSystemKubernetesNode) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.kubernetes.node.name'") + } + return nil +} + +func (m *metadataSystemKubernetesPod) IsSet() bool { + return m.Name.IsSet() || m.UID.IsSet() +} + +func (m *metadataSystemKubernetesPod) Reset() { + m.Name.Reset() + m.UID.Reset() +} + +func (m *metadataSystemKubernetesPod) validate() error { + if !m.IsSet() { + return nil + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.kubernetes.pod.name'") + } + if utf8.RuneCountInString(m.UID.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.system.kubernetes.pod.uid'") + } + return nil +} + +func (m *metadataUser) IsSet() bool { + return m.ID.IsSet() || m.Email.IsSet() || m.Name.IsSet() +} + +func (m *metadataUser) Reset() { + m.ID.Reset() + m.Email.Reset() + m.Name.Reset() +} + +func (m *metadataUser) validate() error { + if !m.IsSet() { + return nil + } + switch t := m.ID.Val.(type) { + case string: + if utf8.RuneCountInString(t) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.user.id'") + } + case json.Number: + if _, err := t.Int64(); err != nil { + return fmt.Errorf("validation rule 'types(string;int)' violated for 'metadata.user.id'") + } + case int: + case nil: + default: + return fmt.Errorf("validation rule 'types(string;int)' violated for 'metadata.user.id'") + } + if utf8.RuneCountInString(m.Email.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.user.email'") + } + if utf8.RuneCountInString(m.Name.Val) > 1024 { + return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.user.username'") + } + return nil +} + +func (m *metadataWithKey) IsSet() bool { + return m.Metadata.IsSet() +} + +func (m *metadataWithKey) Reset() { + m.Metadata.Reset() +} + +func (m *metadataWithKey) validate() error { + if err := m.Metadata.validate(); err != nil { + return err + } + if !m.Metadata.IsSet() { + return fmt.Errorf("'metadata' required") + } + return nil +} diff --git a/model/modeldecoder/v2/model_test.go b/model/modeldecoder/v2/model_test.go new file mode 100644 index 00000000000..7637f5f1ca7 --- /dev/null +++ b/model/modeldecoder/v2/model_test.go @@ -0,0 +1,341 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v2 + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/decoder" + "github.com/elastic/apm-server/model/modeldecoder/typ" + "github.com/elastic/apm-server/tests/loader" + "github.com/elastic/beats/v7/libbeat/common" +) + +var testMinValidMetadata = ` +{"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}` + +func TestIsSet(t *testing.T) { + inp := `{"cloud":{"availability_zone":"eu-west-3","instance":{"id":"1234"}}}` + var m metadata + require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(inp)).Decode(&m)) + assert.True(t, m.IsSet()) + assert.True(t, m.Cloud.IsSet()) + assert.True(t, m.Cloud.AvailabilityZone.IsSet()) + assert.True(t, m.Cloud.Instance.ID.IsSet()) + assert.False(t, m.Cloud.Instance.Name.IsSet()) +} +func TestSetReset(t *testing.T) { + var m metadataWithKey + inp, err := loader.LoadDataAsStream("../testdata/intake-v2/metadata.ndjson") + require.NoError(t, err) + require.NoError(t, decoder.NewJSONIteratorDecoder(inp).Decode(&m)) + require.True(t, m.IsSet()) + require.True(t, m.Metadata.Cloud.IsSet()) + require.NotEmpty(t, m.Metadata.Labels) + require.True(t, m.Metadata.Process.IsSet()) + require.True(t, m.Metadata.Service.IsSet()) + require.True(t, m.Metadata.System.IsSet()) + require.True(t, m.Metadata.User.IsSet()) + // call Reset and ensure initial state, except for array capacity + m.Reset() + assert.False(t, m.IsSet()) + assert.Equal(t, metadataCloud{}, m.Metadata.Cloud) + assert.Equal(t, metadataService{}, m.Metadata.Service) + assert.Equal(t, metadataSystem{}, m.Metadata.System) + assert.Equal(t, metadataUser{}, m.Metadata.User) + assert.Empty(t, m.Metadata.Labels) + assert.Empty(t, m.Metadata.Process.Pid) + assert.Empty(t, m.Metadata.Process.Ppid) + assert.Empty(t, m.Metadata.Process.Title) + // test that array len is set to zero, but not capacity + assert.Empty(t, m.Metadata.Process.Argv) + assert.Greater(t, cap(m.Metadata.Process.Argv), 0) +} + +func TestValidationRules(t *testing.T) { + type testcase struct { + name string + errorKey string + data string + } + + strBuilder := func(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = '⌘' + } + return string(b) + } + + testMetadata := func(key string, tc testcase) { + // load minimal required data + // set testcase data for given key + var data map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(testMinValidMetadata), &data)) + var keyData map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(tc.data), &keyData)) + data[key] = keyData + // unmarshal data into metdata struct + var m metadata + b, err := json.Marshal(data) + require.NoError(t, err) + require.NoError(t, decoder.NewJSONIteratorDecoder(bytes.NewReader(b)).Decode(&m)) + // run validation and checks + err = m.validate() + if tc.errorKey == "" { + assert.NoError(t, err) + } else { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errorKey) + } + } + + t.Run("user", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "id-string", data: `{"id":"user123"}`}, + {name: "id-int", data: `{"id":44}`}, + {name: "id-float", errorKey: "types", data: `{"id":45.6}`}, + {name: "id-bool", errorKey: "types", data: `{"id":true}`}, + {name: "id-string-max-len", data: `{"id":"` + strBuilder(1024) + `"}`}, + {name: "id-string-max-len", errorKey: "max", data: `{"id":"` + strBuilder(1025) + `"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + testMetadata("user", tc) + }) + } + }) + + t.Run("service", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "name-valid-lower", data: `"name":"abcdefghijklmnopqrstuvwxyz"`}, + {name: "name-valid-upper", data: `"name":"ABCDEFGHIJKLMNOPQRSTUVWXYZ"`}, + {name: "name-valid-digits", data: `"name":"0123456789"`}, + {name: "name-valid-special", data: `"name":"_ -"`}, + {name: "name-asterisk", errorKey: "service.name", data: `"name":"abc*"`}, + {name: "name-dot", errorKey: "service.name", data: `"name":"abc."`}, + } { + t.Run(tc.name, func(t *testing.T) { + tc.data = `{"agent":{"name":"go","version":"1.0"},` + tc.data + `}` + testMetadata("service", tc) + }) + } + }) + + t.Run("labels", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "valid", data: `{"k1":"v1","k2":2.3,"k3":3,"k4":true,"k5":null}`}, + {name: "restricted-type", errorKey: "typesVals", data: `{"k1":{"k2":"v1"}}`}, + {name: "key-dot", errorKey: "patternKeys", data: `{"k.1":"v1"}`}, + {name: "key-asterisk", errorKey: "patternKeys", data: `{"k*1":"v1"}`}, + {name: "key-quotemark", errorKey: "patternKeys", data: `{"k\"1":"v1"}`}, + {name: "max-len", data: `{"k1":"` + strBuilder(1024) + `"}`}, + {name: "max-len-exceeded", errorKey: "maxVals", data: `{"k1":"` + strBuilder(1025) + `"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + testMetadata("labels", tc) + }) + } + }) + + t.Run("max-len", func(t *testing.T) { + // check that `max` on strings is respected on an arbitrary field + for _, tc := range []testcase{ + {name: "title-max-len", data: `{"pid":1,"title":"` + strBuilder(1024) + `"}`}, + {name: "title-max-len-exceeded", errorKey: "max", + data: `{"pid":1,"title":"` + strBuilder(1025) + `"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + testMetadata("process", tc) + }) + } + }) + + t.Run("required", func(t *testing.T) { + // setup: create full metadata struct + typ := reflect.TypeOf(metadata{}) + val := reflect.New(typ) + iterateStruct(typ, val.Elem(), "", true, nil) + metadata := val.Interface().(*metadata) + // test vanilla struct is valid + require.NoError(t, metadata.validate()) + + requiredKeys := map[string]interface{}{ + "cloud.provider": nil, + "process.pid": nil, + "service": nil, + "service.agent": nil, + "service.agent.name": nil, + "service.agent.version": nil, + "service.language.name": nil, + "service.runtime.name": nil, + "service.runtime.version": nil, + "service.name": nil, + } + // iterate through struct and remove every key one by one + iterateStruct(typ, val.Elem(), "", false, func(key string) { + err := metadata.validate() + if _, ok := requiredKeys[key]; ok { + require.Error(t, err, key) + assert.Contains(t, err.Error(), key) + } else { + assert.NoError(t, err, key) + } + }) + }) +} + +func iterateStruct(t reflect.Type, v reflect.Value, key string, init bool, cb func(string)) { + if t.Kind() != reflect.Struct { + panic(fmt.Sprintf("iterateStruct: invalid typ %T", t.Kind())) + } + if key != "" { + key += "." + } + var fKey string + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + if !f.CanSet() { + continue + } + stf := t.Field(i) + fTyp := stf.Type + fKey = fmt.Sprintf("%s%s", key, jsonName(stf)) + if init { + initStruct(fTyp, f, fKey) + continue + } + toZero(fTyp, f, fKey, cb) + } +} + +func toZero(fTyp reflect.Type, f reflect.Value, fKey string, cb func(string)) { + switch k := fTyp.Kind(); k { + case reflect.Map: + orig := f.Interface() + switch val := f.Interface().(type) { + case map[string]interface{}: + var m map[string]interface{} + f.Set(reflect.ValueOf(m)) + case common.MapStr: + var m common.MapStr + f.Set(reflect.ValueOf(m)) + default: + panic(fmt.Sprintf("iterateStruct: unhandled type %T for map", val)) + } + cb(fKey) + f.Set(reflect.ValueOf(orig)) + case reflect.Slice: + val := f.Interface() + switch f.Interface().(type) { + case []string: + var arr []string + f.Set(reflect.ValueOf(arr)) + case []int: + var arr []int + f.Set(reflect.ValueOf(arr)) + } + cb(fKey) + f.Set(reflect.ValueOf(val)) + case reflect.Struct: + switch val := f.Interface().(type) { + case typ.String: + var value typ.String + value.Reset() + f.Set(reflect.ValueOf(value)) + cb(fKey) + f.Set(reflect.ValueOf(val)) + case typ.Int: + var value typ.Int + value.Reset() + f.Set(reflect.ValueOf(value)) + cb(fKey) + f.Set(reflect.ValueOf(val)) + case typ.Interface: + var value typ.Interface + value.Reset() + f.Set(reflect.ValueOf(value)) + cb(fKey) + f.Set(reflect.ValueOf(val)) + default: + iterateStruct(fTyp, f, fKey, false, cb) + } + default: + panic(fmt.Sprintf("iterateStruct: unhandled type %T", k)) + } +} + +func initStruct(fTyp reflect.Type, f reflect.Value, fKey string) { + switch k := fTyp.Kind(); k { + case reflect.Map: + var m interface{} + switch val := f.Interface().(type) { + case map[string]interface{}: + m = map[string]interface{}{"k1": "v1"} + case common.MapStr: + m = common.MapStr{"k1": "v1"} + default: + panic(fmt.Sprintf("iterateStruct: unhandled type %T for map", val)) + } + f.Set(reflect.ValueOf(m)) + case reflect.Slice: + var arr interface{} + switch f.Interface().(type) { + case []string: + arr = []string{"a", "b"} + case []int: + arr = []int{1, 2, 3} + } + f.Set(reflect.ValueOf(arr)) + case reflect.Struct: + switch val := f.Interface().(type) { + case typ.String: + val.Set("teststring") + f.Set(reflect.ValueOf(val)) + case typ.Int: + val.Set(123) + f.Set(reflect.ValueOf(val)) + case typ.Interface: + val.Set("testinterface") + f.Set(reflect.ValueOf(val)) + default: + iterateStruct(fTyp, f, fKey, true, nil) + } + default: + panic(fmt.Sprintf("iterateStruct: unhandled type %T", k)) + } +} + +func jsonName(f reflect.StructField) string { + tag, ok := f.Tag.Lookup("json") + if !ok || tag == "-" { + return "" + } + parts := strings.Split(tag, ",") + if len(parts) == 0 { + return "" + } + return parts[0] +} diff --git a/processor/stream/package_tests/metadata_attrs_test.go b/processor/stream/package_tests/metadata_attrs_test.go index 7fb2f567c0f..80dc4b1d4e5 100644 --- a/processor/stream/package_tests/metadata_attrs_test.go +++ b/processor/stream/package_tests/metadata_attrs_test.go @@ -74,12 +74,12 @@ func metadataProcSetup() *tests.ProcessorSetup { TemplatePaths: []string{ "../../../_meta/fields.common.yml", }, - FullPayloadPath: "../testdata/intake-v2/only-metadata.ndjson", + FullPayloadPath: "../testdata/intake-v2/metadata.ndjson", } } func getMetadataEventAttrs(t *testing.T, prefix string) *tests.Set { - payloadStream, err := loader.LoadDataAsStream("../testdata/intake-v2/only-metadata.ndjson") + payloadStream, err := loader.LoadDataAsStream("../testdata/intake-v2/metadata.ndjson") require.NoError(t, err) var metadata map[string]interface{} @@ -123,7 +123,7 @@ func TestMetadataPayloadAttrsMatchFields(t *testing.T) { func TestMetadataPayloadMatchJsonSchema(t *testing.T) { metadataProcSetup().AttrsMatchJsonSchema(t, getMetadataEventAttrs(t, ""), - tests.NewSet(tests.Group("labels")), + tests.NewSet(tests.Group("labels"), "system.ip"), nil, ) } diff --git a/testdata/intake-v2/metadata.ndjson b/testdata/intake-v2/metadata.ndjson new file mode 100644 index 00000000000..bccc533ed42 --- /dev/null +++ b/testdata/intake-v2/metadata.ndjson @@ -0,0 +1 @@ +{"metadata":{"service":{"name":"1234_service-12a3","node":{"configured_name":"node-123"},"version":"5.1.3","environment":"staging","language":{"name":"ecmascript","version":"8"},"runtime":{"name":"node","version":"8.0.0"},"framework":{"name":"Express","version":"1.2.3"},"agent":{"name":"elastic-node","version":"3.14.0","ephemeral_id":"e71be9ac-93b0-44b9-a997-5638f6ccfc36"}},"user":{"id":"123user","username":"bar","email":"bar@user.com"},"system":{"hostname":"prod1.example.com","configured_hostname":"prod1.example.com","detected_hostname":"prod1.example.com","architecture":"x64","ip":"127.0.0.1","platform":"darwin","container":{"id":"container-id"},"kubernetes":{"namespace":"namespace1","pod":{"uid":"pod-uid","name":"pod-name"},"node":{"name":"node-name"}}},"process":{"pid":1234,"ppid":6789,"title":"node","argv":["node","server.js"]},"labels":{"tag0":null,"tag1":"one","tag2":2},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region"}}} diff --git a/testdata/intake-v2/only-metadata.ndjson b/testdata/intake-v2/only-metadata.ndjson deleted file mode 100644 index 98f74e717c4..00000000000 --- a/testdata/intake-v2/only-metadata.ndjson +++ /dev/null @@ -1 +0,0 @@ -{"metadata": {"process": {"ppid": 6789, "pid": 1234, "argv": ["node", "server.js"], "title": "node"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "configured_hostname": "foo", "detected_hostname": "myhostname" ,"architecture": "x64", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}}, "service": {"name": "1234_service-12a3","node":{"configured_name":"abc-xyz"},"language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node", "ephemeral_id": "123abcdef"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "user": {"id": "99","username": "foo","email": "foo@example.com"},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region"}}} diff --git a/testdata/intake-v3/metadata.ndjson b/testdata/intake-v3/metadata.ndjson new file mode 100644 index 00000000000..e34e62e4996 --- /dev/null +++ b/testdata/intake-v3/metadata.ndjson @@ -0,0 +1 @@ +{"m": {"se": {"n": "apm-a-rum-test-e2e-general-usecase","ve": "0.0.1","en": "prod","a": {"n": "js-base","ve": "4.8.1"},"ru": {"n": "v8","ve": "8.0"},"la": {"n": "javascript","ve": "6"},"fw": {"n": "angular","ve": "2"}},"u": {"id": 123,"em": "user@email.com","un": "John Doe"},"l": {"testTagKey": "testTagValue"}}} \ No newline at end of file diff --git a/tests/system/test_requests.py b/tests/system/test_requests.py index 5a1caef8a44..27bf0b8ccb3 100644 --- a/tests/system/test_requests.py +++ b/tests/system/test_requests.py @@ -200,7 +200,7 @@ def test_rate_limit_small_hit(self): def test_rate_limit_only_metadata(self): # all requests from the same ip # no events, batch size 10 => 10+1 events per requ - codes = self.fire_events("only-metadata.ndjson", 8) + codes = self.fire_events("metadata.ndjson", 8) assert set(codes.keys()) == set([202, 429]), codes assert codes[429] == 3, codes assert codes[202] == 5, codes From f84db7f11770f451162a21912a0f55da7043c8bf Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 7 Sep 2020 13:43:07 +0200 Subject: [PATCH 2/6] Incorporate PR comments * switch from pkg name `type` to `nullable` * rename Decoder methods * simplify v2 validation test logic * minor refactorings --- main.go | 2 +- .../modeldecoder/{ => generator}/cmd/main.go | 4 +- model/modeldecoder/generator/generator.go | 34 ++-- .../{typ/typ.go => nullable/nullable.go} | 9 +- .../typ_test.go => nullable/nullable_test.go} | 2 +- model/modeldecoder/rumv3/model.go | 30 ++-- model/modeldecoder/rumv3/model_generated.go | 4 +- model/modeldecoder/v2/decoder.go | 68 ++++---- model/modeldecoder/v2/decoder_test.go | 42 ++--- model/modeldecoder/v2/model.go | 90 +++++----- model/modeldecoder/v2/model_generated.go | 28 +-- model/modeldecoder/v2/model_test.go | 162 ++++++------------ 12 files changed, 197 insertions(+), 278 deletions(-) rename model/modeldecoder/{ => generator}/cmd/main.go (94%) rename model/modeldecoder/{typ/typ.go => nullable/nullable.go} (88%) rename model/modeldecoder/{typ/typ_test.go => nullable/nullable_test.go} (99%) diff --git a/main.go b/main.go index 82d5b49a94b..6b539245033 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,7 @@ package main //go:generate go run script/inline_schemas/inline_schemas.go -//go:generate go run model/modeldecoder/cmd/main.go +//go:generate go run model/modeldecoder/generator/cmd/main.go import ( "os" diff --git a/model/modeldecoder/cmd/main.go b/model/modeldecoder/generator/cmd/main.go similarity index 94% rename from model/modeldecoder/cmd/main.go rename to model/modeldecoder/generator/cmd/main.go index b1f081f305a..c29bf2aa799 100644 --- a/model/modeldecoder/cmd/main.go +++ b/model/modeldecoder/generator/cmd/main.go @@ -33,7 +33,7 @@ const ( var ( importPath = filepath.Join(basePath, modeldecoderPath) - typPath = filepath.Join(importPath, "typ") + typPath = filepath.Join(importPath, "nullable") ) func main() { @@ -43,7 +43,7 @@ func main() { func genV2Models() { pkg := "v2" - rootObjs := []string{"metadataNoKey", "metadataWithKey"} + rootObjs := []string{"metadataRoot"} out := filepath.Join(modeldecoderPath, pkg, "model_generated.go") gen, err := generator.NewGenerator(importPath, pkg, typPath, rootObjs) if err != nil { diff --git a/model/modeldecoder/generator/generator.go b/model/modeldecoder/generator/generator.go index 75b72bce16f..c21394e84cf 100644 --- a/model/modeldecoder/generator/generator.go +++ b/model/modeldecoder/generator/generator.go @@ -24,6 +24,7 @@ import ( "go/ast" "go/token" "go/types" + "path" "path/filepath" "reflect" "sort" @@ -46,9 +47,9 @@ type Generator struct { structTypes structTypes // keep track of already processed types in case one type is // referenced multiple times - processedTypes map[string]interface{} + processedTypes map[string]struct{} - typString, typInt, typInterface string + nullableString, nullableInt, nullableInterface string } // NewGenerator takes an importPath and the package name for which @@ -58,7 +59,7 @@ type Generator struct { // directly or indirectly by any of the root types. func NewGenerator(importPath string, pkg string, typPath string, root []string) (*Generator, error) { - loaded, err := loadPackage(fmt.Sprintf("%s/%s", importPath, pkg)) + loaded, err := loadPackage(path.Join(importPath, pkg)) if err != nil { return nil, err } @@ -67,13 +68,13 @@ func NewGenerator(importPath string, pkg string, typPath string, return nil, err } g := Generator{ - pkgName: loaded.Types.Name(), - structTypes: structTypes, - rootObjs: make(map[string]structType, len(root)), - processedTypes: make(map[string]interface{}), - typString: fmt.Sprintf("%s.String", typPath), - typInt: fmt.Sprintf("%s.Int", typPath), - typInterface: fmt.Sprintf("%s.Interface", typPath), + pkgName: loaded.Types.Name(), + structTypes: structTypes, + rootObjs: make(map[string]structType, len(root)), + processedTypes: make(map[string]struct{}), + nullableString: fmt.Sprintf("%s.String", typPath), + nullableInt: fmt.Sprintf("%s.Int", typPath), + nullableInterface: fmt.Sprintf("%s.Interface", typPath), } for _, r := range root { rootObjPath := fmt.Sprintf("%s.%s", filepath.Join(importPath, pkg), r) @@ -89,6 +90,8 @@ func NewGenerator(importPath string, pkg string, typPath string, // Generate writes generated methods to the buffer func (g *Generator) Generate() (bytes.Buffer, error) { fmt.Fprintf(&g.buf, ` +// Code generated by "modeldecoder/generator". DO NOT EDIT. + package %s import ( @@ -136,7 +139,7 @@ func (g *Generator) generate(st structType, key string) error { if _, ok := g.processedTypes[st.name]; ok { return nil } - g.processedTypes[st.name] = nil + g.processedTypes[st.name] = struct{}{} if err := g.generateIsSet(st, key); err != nil { return err } @@ -340,7 +343,7 @@ if utf8.RuneCountInString(t) > %s{ } fmt.Fprintf(&g.buf, ` default: - return fmt.Errorf("validation rule '%s(%s)' violated for '%s' for key " + k) + return fmt.Errorf("validation rule '%s(%s)' violated for '%s' for key %%s",k) } `[1:], ruleTypesVals, types, flattenedName) } @@ -354,7 +357,7 @@ default: case *types.Struct: switch f.typ.String() { //TODO(simitt): can these type checks be more generic? - case g.typString: + case g.nullableString: for _, rule := range sortedRules { val := parts[rule] switch rule { @@ -380,7 +383,7 @@ if !%s.MatchString(m.%s.Val){ return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) } } - case g.typInt: + case g.nullableInt: for _, rule := range sortedRules { val := parts[rule] switch rule { @@ -400,7 +403,7 @@ if m.%s.Val > %s{ return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName) } } - case g.typInterface: + case g.nullableInterface: var required bool if _, ok := parts[ruleRequired]; ok { required = true @@ -482,6 +485,7 @@ if !m.%s.IsSet(){ `[1:]) return nil } + func jsonName(f structField) string { parts := parseTag(f.tag, "json") if len(parts) == 0 { diff --git a/model/modeldecoder/typ/typ.go b/model/modeldecoder/nullable/nullable.go similarity index 88% rename from model/modeldecoder/typ/typ.go rename to model/modeldecoder/nullable/nullable.go index 0f9fa3e1c9d..9d3940565bd 100644 --- a/model/modeldecoder/typ/typ.go +++ b/model/modeldecoder/nullable/nullable.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package typ +package nullable import ( "unsafe" @@ -24,7 +24,7 @@ import ( ) func init() { - jsoniter.RegisterTypeDecoderFunc("typ.String", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + jsoniter.RegisterTypeDecoderFunc("nullable.String", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { switch iter.WhatIsNext() { case jsoniter.NilValue: iter.ReadNil() @@ -34,7 +34,7 @@ func init() { } (*((*String)(ptr))).isSet = true }) - jsoniter.RegisterTypeDecoderFunc("typ.Int", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + jsoniter.RegisterTypeDecoderFunc("nullable.Int", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { switch iter.WhatIsNext() { case jsoniter.NilValue: iter.ReadNil() @@ -44,7 +44,7 @@ func init() { } (*((*Int)(ptr))).isSet = true }) - jsoniter.RegisterTypeDecoderFunc("typ.Interface", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + jsoniter.RegisterTypeDecoderFunc("nullable.Interface", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { switch iter.WhatIsNext() { case jsoniter.NilValue: iter.ReadNil() @@ -116,6 +116,7 @@ func (v *Int) Reset() { v.isNotNil = false } +// TODO(simitt): follow up on https://github.com/elastic/apm-server/pull/4154#discussion_r484166721 type Interface struct { Val interface{} `json:"val,omitempty"` isSet, isNotNil bool diff --git a/model/modeldecoder/typ/typ_test.go b/model/modeldecoder/nullable/nullable_test.go similarity index 99% rename from model/modeldecoder/typ/typ_test.go rename to model/modeldecoder/nullable/nullable_test.go index 509c189298c..a89156f1005 100644 --- a/model/modeldecoder/typ/typ_test.go +++ b/model/modeldecoder/nullable/nullable_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package typ +package nullable import ( "strings" diff --git a/model/modeldecoder/rumv3/model.go b/model/modeldecoder/rumv3/model.go index 5b9254e4709..18d7035616b 100644 --- a/model/modeldecoder/rumv3/model.go +++ b/model/modeldecoder/rumv3/model.go @@ -20,7 +20,7 @@ package rumv3 import ( "regexp" - "github.com/elastic/apm-server/model/modeldecoder/typ" + "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/beats/v7/libbeat/common" ) @@ -39,42 +39,42 @@ type metadata struct { // metadataService holds information about where the data was collected type metadataService struct { Agent metadataServiceAgent `json:"a" validate:"required"` - Environment typ.String `json:"en" validate:"max=1024"` + Environment nullable.String `json:"en" validate:"max=1024"` Framework MetadataServiceFramework `json:"fw"` Language metadataServiceLanguage `json:"la"` - Name typ.String `json:"n" validate:"required,max=1024,pattern=alphaNumericExtRegex"` + Name nullable.String `json:"n" validate:"required,max=1024,pattern=alphaNumericExtRegex"` Runtime metadataServiceRuntime `json:"ru"` - Version typ.String `json:"ve" validate:"max=1024"` + Version nullable.String `json:"ve" validate:"max=1024"` } //metadataServiceAgent has a version and a name type metadataServiceAgent struct { - Name typ.String `json:"n" validate:"required,max=1024"` - Version typ.String `json:"ve" validate:"required,max=1024"` + Name nullable.String `json:"n" validate:"required,max=1024"` + Version nullable.String `json:"ve" validate:"required,max=1024"` } //MetadataServiceFramework has a version and name type MetadataServiceFramework struct { - Name typ.String `json:"n" validate:"max=1024"` - Version typ.String `json:"ve" validate:"max=1024"` + Name nullable.String `json:"n" validate:"max=1024"` + Version nullable.String `json:"ve" validate:"max=1024"` } //MetadataLanguage has a version and name type metadataServiceLanguage struct { - Name typ.String `json:"n" validate:"required,max=1024"` - Version typ.String `json:"ve" validate:"max=1024"` + Name nullable.String `json:"n" validate:"required,max=1024"` + Version nullable.String `json:"ve" validate:"max=1024"` } //MetadataRuntime has a version and name type metadataServiceRuntime struct { - Name typ.String `json:"n" validate:"required,max=1024"` - Version typ.String `json:"ve" validate:"required,max=1024"` + Name nullable.String `json:"n" validate:"required,max=1024"` + Version nullable.String `json:"ve" validate:"required,max=1024"` } type metadataUser struct { - ID typ.Interface `json:"id" validate:"max=1024,types=string;int"` - Email typ.String `json:"em" validate:"max=1024"` - Name typ.String `json:"un" validate:"max=1024"` + ID nullable.Interface `json:"id" validate:"max=1024,types=string;int"` + Email nullable.String `json:"em" validate:"max=1024"` + Name nullable.String `json:"un" validate:"max=1024"` } type metadataWithKey struct { diff --git a/model/modeldecoder/rumv3/model_generated.go b/model/modeldecoder/rumv3/model_generated.go index 942e1f5cb83..42050f2c69c 100644 --- a/model/modeldecoder/rumv3/model_generated.go +++ b/model/modeldecoder/rumv3/model_generated.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// Code generated by "modeldecoder/generator". DO NOT EDIT. + package rumv3 import ( @@ -70,7 +72,7 @@ func (m *metadata) validate() error { case bool: case json.Number: default: - return fmt.Errorf("validation rule 'typesVals(string;bool;number)' violated for 'm.l' for key " + k) + return fmt.Errorf("validation rule 'typesVals(string;bool;number)' violated for 'm.l' for key %s", k) } } if err := m.Service.validate(); err != nil { diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index fb9e047cdd8..dcb1e3be615 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -28,57 +28,51 @@ import ( ) func init() { - metadataWithKeyPool.New = func() interface{} { - return &metadataWithKey{} - } - metadataNoKeyPool.New = func() interface{} { - return &metadataNoKey{} + metadataRootPool.New = func() interface{} { + return &metadataRoot{} } + // metadataNoKeyPool.New = func() interface{} { + // return &metadataNoKey{} + // } } -var metadataWithKeyPool, metadataNoKeyPool sync.Pool - -func fetchMetadataWithKey() *metadataWithKey { - return metadataWithKeyPool.Get().(*metadataWithKey) -} -func releaseMetadataWithKey(m *metadataWithKey) { - m.Reset() - metadataWithKeyPool.Put(m) -} +var metadataRootPool sync.Pool -func fetchMetadataNoKey() *metadataNoKey { - return metadataNoKeyPool.Get().(*metadataNoKey) +func fetchMetadataRoot() *metadataRoot { + return metadataRootPool.Get().(*metadataRoot) } -func releaseMetadataNoKey(m *metadataNoKey) { +func releaseMetadataRoot(m *metadataRoot) { m.Reset() - metadataNoKeyPool.Put(m) + metadataRootPool.Put(m) } -// DecodeProfileMetadata uses the given decoder to create the input models, +// DecodeMetadata uses the given decoder to create the input models, // then runs the defined validations on the input models // and finally maps the values fom the input model to the given *model.Metadata instance -func DecodeProfileMetadata(d decoder.Decoder, out *model.Metadata) error { - var err error - m := fetchMetadataNoKey() - defer releaseMetadataNoKey(m) - if err = d.Decode(&m.Metadata); err != nil { - return fmt.Errorf("decode error %w", err) - } - if err := m.validate(); err != nil { - return fmt.Errorf("validation error %w", err) - } - mapToMetadataModel(&m.Metadata, out) - return nil +// +// DecodeMetadata should be used when the underlying byte stream does not contain the +// `metadata` key, but only the metadata. +func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { + return decode(func(m *metadataRoot) error { + return d.Decode(&m.Metadata) + }, out) } -// DecodeMetadata uses the given decoder to create the input models, +// DecodeNestedMetadata uses the given decoder to create the input models, // then runs the defined validations on the input models // and finally maps the values fom the input model to the given *model.Metadata instance -func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { - var err error - m := fetchMetadataWithKey() - defer releaseMetadataWithKey(m) - if err = d.Decode(m); err != nil { +// +// DecodeNestedMetadata should be used when the underlying byte stream does start with the `metadata` key +func DecodeNestedMetadata(d decoder.Decoder, out *model.Metadata) error { + return decode(func(m *metadataRoot) error { + return d.Decode(m) + }, out) +} + +func decode(decoderFn func(m *metadataRoot) error, out *model.Metadata) error { + m := fetchMetadataRoot() + defer releaseMetadataRoot(m) + if err := decoderFn(m); err != nil { return fmt.Errorf("decode error %w", err) } if err := m.validate(); err != nil { diff --git a/model/modeldecoder/v2/decoder_test.go b/model/modeldecoder/v2/decoder_test.go index 677351cbe44..75be981af7d 100644 --- a/model/modeldecoder/v2/decoder_test.go +++ b/model/modeldecoder/v2/decoder_test.go @@ -30,17 +30,11 @@ import ( func TestResetModelOnRelease(t *testing.T) { inp := `{"metadata":{"service":{"name":"service-a"}}}` - m := fetchMetadataWithKey() + m := fetchMetadataRoot() require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(inp)).Decode(m)) require.True(t, m.IsSet()) - releaseMetadataWithKey(m) + releaseMetadataRoot(m) assert.False(t, m.IsSet()) - - mNoKey := fetchMetadataNoKey() - require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(inp)).Decode(mNoKey)) - require.True(t, mNoKey.IsSet()) - releaseMetadataNoKey(mNoKey) - assert.False(t, mNoKey.IsSet()) } func TestDecodeProfileMetadata(t *testing.T) { @@ -49,7 +43,7 @@ func TestDecodeProfileMetadata(t *testing.T) { decodeProfileMetadata := func(out *model.Metadata) { dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) - require.NoError(t, DecodeProfileMetadata(dec, out)) + require.NoError(t, DecodeMetadata(dec, out)) } t.Run("fetch-release", func(t *testing.T) { @@ -61,11 +55,11 @@ func TestDecodeProfileMetadata(t *testing.T) { // test behavior by overriding the new method, occupying an existing // instance and counting how often the New method is called var newCount, expectedNewCount int - origNew := metadataNoKeyPool.New - defer func() { metadataNoKeyPool.New = origNew }() - metadataNoKeyPool.New = func() interface{} { + origNew := metadataRootPool.New + defer func() { metadataRootPool.New = origNew }() + metadataRootPool.New = func() interface{} { newCount++ - return &metadataNoKey{} + return &metadataRoot{} } var out model.Metadata // on the first call align the expected with the current new count @@ -76,7 +70,7 @@ func TestDecodeProfileMetadata(t *testing.T) { decodeProfileMetadata(&out) assert.Equal(t, expectedNewCount, newCount) // force a new instance on the next decoder call - fetchMetadataNoKey() + fetchMetadataRoot() decodeProfileMetadata(&out) assert.Equal(t, expectedNewCount+1, newCount) }) @@ -88,7 +82,7 @@ func TestDecodeProfileMetadata(t *testing.T) { Name: "user-service", Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) - err := DecodeProfileMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) + err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) require.Error(t, err) assert.Contains(t, err.Error(), "decode") }) @@ -96,7 +90,7 @@ func TestDecodeProfileMetadata(t *testing.T) { t.Run("validate", func(t *testing.T) { inp := `{}` var out model.Metadata - err := DecodeProfileMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) + err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) require.Error(t, err) assert.Contains(t, err.Error(), "validation") }) @@ -109,7 +103,7 @@ func TestDecodeMetadata(t *testing.T) { decodeMetadata := func(out *model.Metadata) { dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) - require.NoError(t, DecodeMetadata(dec, out)) + require.NoError(t, DecodeNestedMetadata(dec, out)) } t.Run("fetch-release", func(t *testing.T) { @@ -121,11 +115,11 @@ func TestDecodeMetadata(t *testing.T) { // test behavior by overriding the new method, occupying an existing // instance and counting how often the New method is called var newCount, expectedNewCount int - origNew := metadataWithKeyPool.New - defer func() { metadataWithKeyPool.New = origNew }() - metadataWithKeyPool.New = func() interface{} { + origNew := metadataRootPool.New + defer func() { metadataRootPool.New = origNew }() + metadataRootPool.New = func() interface{} { newCount++ - return &metadataWithKey{} + return &metadataRoot{} } var out model.Metadata // on the first call align the expected with the current new count @@ -136,7 +130,7 @@ func TestDecodeMetadata(t *testing.T) { decodeMetadata(&out) assert.Equal(t, expectedNewCount, newCount) // force a new instance on the next decoder call - fetchMetadataWithKey() + fetchMetadataRoot() decodeMetadata(&out) assert.Equal(t, expectedNewCount+1, newCount) }) @@ -148,7 +142,7 @@ func TestDecodeMetadata(t *testing.T) { Name: "user-service", Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) - err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) + err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) require.Error(t, err) assert.Contains(t, err.Error(), "decode") }) @@ -156,7 +150,7 @@ func TestDecodeMetadata(t *testing.T) { t.Run("validate", func(t *testing.T) { inp := `{}` var out model.Metadata - err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) + err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) require.Error(t, err) assert.Contains(t, err.Error(), "validation") }) diff --git a/model/modeldecoder/v2/model.go b/model/modeldecoder/v2/model.go index df8700505f3..86743315af0 100644 --- a/model/modeldecoder/v2/model.go +++ b/model/modeldecoder/v2/model.go @@ -20,7 +20,7 @@ package v2 import ( "regexp" - "github.com/elastic/apm-server/model/modeldecoder/typ" + "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/beats/v7/libbeat/common" ) @@ -40,119 +40,115 @@ type metadata struct { type metadataCloud struct { Account metadataCloudAccount `json:"account"` - AvailabilityZone typ.String `json:"availability_zone" validate:"max=1024"` + AvailabilityZone nullable.String `json:"availability_zone" validate:"max=1024"` Instance metadataCloudInstance `json:"instance"` Machine metadataCloudMachine `json:"machine"` Project metadataCloudProject `json:"project"` - Provider typ.String `json:"provider" validate:"required,max=1024"` - Region typ.String `json:"region" validate:"max=1024"` + Provider nullable.String `json:"provider" validate:"required,max=1024"` + Region nullable.String `json:"region" validate:"max=1024"` } type metadataCloudAccount struct { - ID typ.String `json:"id" validate:"max=1024"` - Name typ.String `json:"name" validate:"max=1024"` + ID nullable.String `json:"id" validate:"max=1024"` + Name nullable.String `json:"name" validate:"max=1024"` } type metadataCloudInstance struct { - ID typ.String `json:"id" validate:"max=1024"` - Name typ.String `json:"name" validate:"max=1024"` + ID nullable.String `json:"id" validate:"max=1024"` + Name nullable.String `json:"name" validate:"max=1024"` } type metadataCloudMachine struct { - Type typ.String `json:"type" validate:"max=1024"` + Type nullable.String `json:"type" validate:"max=1024"` } type metadataCloudProject struct { - ID typ.String `json:"id" validate:"max=1024"` - Name typ.String `json:"name" validate:"max=1024"` + ID nullable.String `json:"id" validate:"max=1024"` + Name nullable.String `json:"name" validate:"max=1024"` } type metadataProcess struct { - Argv []string `json:"argv"` - Pid typ.Int `json:"pid" validate:"required"` - Ppid typ.Int `json:"ppid"` - Title typ.String `json:"title" validate:"max=1024"` + Argv []string `json:"argv"` + Pid nullable.Int `json:"pid" validate:"required"` + Ppid nullable.Int `json:"ppid"` + Title nullable.String `json:"title" validate:"max=1024"` } type metadataService struct { Agent metadataServiceAgent `json:"agent" validate:"required"` - Environment typ.String `json:"environment" validate:"max=1024"` + Environment nullable.String `json:"environment" validate:"max=1024"` Framework metadataServiceFramework `json:"framework"` Language metadataServiceLanguage `json:"language"` - Name typ.String `json:"name" validate:"required,max=1024,pattern=alphaNumericExtRegex"` + Name nullable.String `json:"name" validate:"required,max=1024,pattern=alphaNumericExtRegex"` Node metadataServiceNode `json:"node"` Runtime metadataServiceRuntime `json:"runtime"` - Version typ.String `json:"version" validate:"max=1024"` + Version nullable.String `json:"version" validate:"max=1024"` } type metadataServiceAgent struct { - EphemeralID typ.String `json:"ephemeral_id" validate:"max=1024"` - Name typ.String `json:"name" validate:"required,max=1024"` - Version typ.String `json:"version" validate:"required,max=1024"` + EphemeralID nullable.String `json:"ephemeral_id" validate:"max=1024"` + Name nullable.String `json:"name" validate:"required,max=1024"` + Version nullable.String `json:"version" validate:"required,max=1024"` } type metadataServiceFramework struct { - Name typ.String `json:"name" validate:"max=1024"` - Version typ.String `json:"version" validate:"max=1024"` + Name nullable.String `json:"name" validate:"max=1024"` + Version nullable.String `json:"version" validate:"max=1024"` } type metadataServiceLanguage struct { - Name typ.String `json:"name" validate:"required,max=1024"` - Version typ.String `json:"version" validate:"max=1024"` + Name nullable.String `json:"name" validate:"required,max=1024"` + Version nullable.String `json:"version" validate:"max=1024"` } type metadataServiceNode struct { - Name typ.String `json:"configured_name" validate:"max=1024"` + Name nullable.String `json:"configured_name" validate:"max=1024"` } type metadataServiceRuntime struct { - Name typ.String `json:"name" validate:"required,max=1024"` - Version typ.String `json:"version" validate:"required,max=1024"` + Name nullable.String `json:"name" validate:"required,max=1024"` + Version nullable.String `json:"version" validate:"required,max=1024"` } type metadataSystem struct { - Architecture typ.String `json:"architecture" validate:"max=1024"` - ConfiguredHostname typ.String `json:"configured_hostname" validate:"max=1024"` + Architecture nullable.String `json:"architecture" validate:"max=1024"` + ConfiguredHostname nullable.String `json:"configured_hostname" validate:"max=1024"` Container metadataSystemContainer `json:"container"` - DetectedHostname typ.String `json:"detected_hostname" validate:"max=1024"` - HostnameDeprecated typ.String `json:"hostname" validate:"max=1024"` - IP typ.String `json:"ip"` + DetectedHostname nullable.String `json:"detected_hostname" validate:"max=1024"` + HostnameDeprecated nullable.String `json:"hostname" validate:"max=1024"` + IP nullable.String `json:"ip"` Kubernetes metadataSystemKubernetes `json:"kubernetes"` - Platform typ.String `json:"platform" validate:"max=1024"` + Platform nullable.String `json:"platform" validate:"max=1024"` } type metadataSystemContainer struct { // `id` is the only field in `system.container`, // if `system.container:{}` is sent, it should be considered valid // if additional attributes are defined in the future, add the required tag - ID typ.String `json:"id"` //validate:"required" + ID nullable.String `json:"id"` //validate:"required" } type metadataSystemKubernetes struct { - Namespace typ.String `json:"namespace" validate:"max=1024"` + Namespace nullable.String `json:"namespace" validate:"max=1024"` Node metadataSystemKubernetesNode `json:"node"` Pod metadataSystemKubernetesPod `json:"pod"` } type metadataSystemKubernetesNode struct { - Name typ.String `json:"name" validate:"max=1024"` + Name nullable.String `json:"name" validate:"max=1024"` } type metadataSystemKubernetesPod struct { - Name typ.String `json:"name" validate:"max=1024"` - UID typ.String `json:"uid" validate:"max=1024"` + Name nullable.String `json:"name" validate:"max=1024"` + UID nullable.String `json:"uid" validate:"max=1024"` } type metadataUser struct { - ID typ.Interface `json:"id,omitempty" validate:"max=1024,types=string;int"` - Email typ.String `json:"email" validate:"max=1024"` - Name typ.String `json:"username" validate:"max=1024"` + ID nullable.Interface `json:"id,omitempty" validate:"max=1024,types=string;int"` + Email nullable.String `json:"email" validate:"max=1024"` + Name nullable.String `json:"username" validate:"max=1024"` } -type metadataWithKey struct { +type metadataRoot struct { Metadata metadata `json:"metadata" validate:"required"` } - -type metadataNoKey struct { - Metadata metadata `validate:"required"` -} diff --git a/model/modeldecoder/v2/model_generated.go b/model/modeldecoder/v2/model_generated.go index 1de9cfe99a1..f63197ed387 100644 --- a/model/modeldecoder/v2/model_generated.go +++ b/model/modeldecoder/v2/model_generated.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// Code generated by "modeldecoder/generator". DO NOT EDIT. + package v2 import ( @@ -23,15 +25,15 @@ import ( "unicode/utf8" ) -func (m *metadataNoKey) IsSet() bool { +func (m *metadataRoot) IsSet() bool { return m.Metadata.IsSet() } -func (m *metadataNoKey) Reset() { +func (m *metadataRoot) Reset() { m.Metadata.Reset() } -func (m *metadataNoKey) validate() error { +func (m *metadataRoot) validate() error { if err := m.Metadata.validate(); err != nil { return err } @@ -76,7 +78,7 @@ func (m *metadata) validate() error { case bool: case json.Number: default: - return fmt.Errorf("validation rule 'typesVals(string;bool;number)' violated for 'metadata.labels' for key " + k) + return fmt.Errorf("validation rule 'typesVals(string;bool;number)' violated for 'metadata.labels' for key %s", k) } } if err := m.Process.validate(); err != nil { @@ -590,21 +592,3 @@ func (m *metadataUser) validate() error { } return nil } - -func (m *metadataWithKey) IsSet() bool { - return m.Metadata.IsSet() -} - -func (m *metadataWithKey) Reset() { - m.Metadata.Reset() -} - -func (m *metadataWithKey) validate() error { - if err := m.Metadata.validate(); err != nil { - return err - } - if !m.Metadata.IsSet() { - return fmt.Errorf("'metadata' required") - } - return nil -} diff --git a/model/modeldecoder/v2/model_test.go b/model/modeldecoder/v2/model_test.go index 7637f5f1ca7..7acc59821e5 100644 --- a/model/modeldecoder/v2/model_test.go +++ b/model/modeldecoder/v2/model_test.go @@ -21,6 +21,7 @@ import ( "bytes" "encoding/json" "fmt" + "os" "reflect" "strings" "testing" @@ -29,8 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/apm-server/decoder" - "github.com/elastic/apm-server/model/modeldecoder/typ" - "github.com/elastic/apm-server/tests/loader" + "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/beats/v7/libbeat/common" ) @@ -47,11 +47,12 @@ func TestIsSet(t *testing.T) { assert.True(t, m.Cloud.Instance.ID.IsSet()) assert.False(t, m.Cloud.Instance.Name.IsSet()) } + func TestSetReset(t *testing.T) { - var m metadataWithKey - inp, err := loader.LoadDataAsStream("../testdata/intake-v2/metadata.ndjson") + var m metadataRoot + r, err := os.Open("../../../testdata/intake-v2/metadata.ndjson") require.NoError(t, err) - require.NoError(t, decoder.NewJSONIteratorDecoder(inp).Decode(&m)) + require.NoError(t, decoder.NewJSONIteratorDecoder(r).Decode(&m)) require.True(t, m.IsSet()) require.True(t, m.Metadata.Cloud.IsSet()) require.NotEmpty(t, m.Metadata.Labels) @@ -174,14 +175,44 @@ func TestValidationRules(t *testing.T) { }) t.Run("required", func(t *testing.T) { - // setup: create full metadata struct + // setup: create full metadata struct with arbitrary values set typ := reflect.TypeOf(metadata{}) val := reflect.New(typ) - iterateStruct(typ, val.Elem(), "", true, nil) - metadata := val.Interface().(*metadata) + iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { + var newVal interface{} + switch val := f.Interface().(type) { + case map[string]interface{}: + newVal = map[string]interface{}{"k1": "v1"} + case common.MapStr: + newVal = common.MapStr{"k1": "v1"} + case []string: + newVal = []string{"a", "b"} + case []int: + newVal = []int{1, 2, 3} + case nullable.String: + val.Set("teststring") + newVal = val + case nullable.Int: + val.Set(123) + newVal = val + case nullable.Interface: + val.Set("testinterface") + newVal = val + default: + if f.Type().Kind() == reflect.Struct { + return + } + panic(fmt.Sprintf("initStruct: unhandled type %T", f.Type().Kind())) + } + f.Set(reflect.ValueOf(newVal)) + }) + // test vanilla struct is valid + metadata := val.Interface().(*metadata) require.NoError(t, metadata.validate()) + // iterate through struct, remove every key one by one + // and test that validation behaves as expected requiredKeys := map[string]interface{}{ "cloud.provider": nil, "process.pid": nil, @@ -194,8 +225,10 @@ func TestValidationRules(t *testing.T) { "service.runtime.version": nil, "service.name": nil, } - // iterate through struct and remove every key one by one - iterateStruct(typ, val.Elem(), "", false, func(key string) { + iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { + original := reflect.ValueOf(f.Interface()) + defer f.Set(original) // reset original value + f.Set(reflect.Zero(f.Type())) err := metadata.validate() if _, ok := requiredKeys[key]; ok { require.Error(t, err, key) @@ -207,7 +240,8 @@ func TestValidationRules(t *testing.T) { }) } -func iterateStruct(t reflect.Type, v reflect.Value, key string, init bool, cb func(string)) { +func iterateStruct(v reflect.Value, key string, cb func(f reflect.Value, fKey string)) { + t := v.Type() if t.Kind() != reflect.Struct { panic(fmt.Sprintf("iterateStruct: invalid typ %T", t.Kind())) } @@ -223,109 +257,19 @@ func iterateStruct(t reflect.Type, v reflect.Value, key string, init bool, cb fu stf := t.Field(i) fTyp := stf.Type fKey = fmt.Sprintf("%s%s", key, jsonName(stf)) - if init { - initStruct(fTyp, f, fKey) - continue - } - toZero(fTyp, f, fKey, cb) - } -} -func toZero(fTyp reflect.Type, f reflect.Value, fKey string, cb func(string)) { - switch k := fTyp.Kind(); k { - case reflect.Map: - orig := f.Interface() - switch val := f.Interface().(type) { - case map[string]interface{}: - var m map[string]interface{} - f.Set(reflect.ValueOf(m)) - case common.MapStr: - var m common.MapStr - f.Set(reflect.ValueOf(m)) - default: - panic(fmt.Sprintf("iterateStruct: unhandled type %T for map", val)) - } - cb(fKey) - f.Set(reflect.ValueOf(orig)) - case reflect.Slice: - val := f.Interface() - switch f.Interface().(type) { - case []string: - var arr []string - f.Set(reflect.ValueOf(arr)) - case []int: - var arr []int - f.Set(reflect.ValueOf(arr)) - } - cb(fKey) - f.Set(reflect.ValueOf(val)) - case reflect.Struct: - switch val := f.Interface().(type) { - case typ.String: - var value typ.String - value.Reset() - f.Set(reflect.ValueOf(value)) - cb(fKey) - f.Set(reflect.ValueOf(val)) - case typ.Int: - var value typ.Int - value.Reset() - f.Set(reflect.ValueOf(value)) - cb(fKey) - f.Set(reflect.ValueOf(val)) - case typ.Interface: - var value typ.Interface - value.Reset() - f.Set(reflect.ValueOf(value)) - cb(fKey) - f.Set(reflect.ValueOf(val)) - default: - iterateStruct(fTyp, f, fKey, false, cb) + if fTyp.Kind() == reflect.Struct { + switch f.Interface().(type) { + case nullable.String, nullable.Int, nullable.Interface: + default: + iterateStruct(f, fKey, cb) + } } - default: - panic(fmt.Sprintf("iterateStruct: unhandled type %T", k)) + cb(f, fKey) } } -func initStruct(fTyp reflect.Type, f reflect.Value, fKey string) { - switch k := fTyp.Kind(); k { - case reflect.Map: - var m interface{} - switch val := f.Interface().(type) { - case map[string]interface{}: - m = map[string]interface{}{"k1": "v1"} - case common.MapStr: - m = common.MapStr{"k1": "v1"} - default: - panic(fmt.Sprintf("iterateStruct: unhandled type %T for map", val)) - } - f.Set(reflect.ValueOf(m)) - case reflect.Slice: - var arr interface{} - switch f.Interface().(type) { - case []string: - arr = []string{"a", "b"} - case []int: - arr = []int{1, 2, 3} - } - f.Set(reflect.ValueOf(arr)) - case reflect.Struct: - switch val := f.Interface().(type) { - case typ.String: - val.Set("teststring") - f.Set(reflect.ValueOf(val)) - case typ.Int: - val.Set(123) - f.Set(reflect.ValueOf(val)) - case typ.Interface: - val.Set("testinterface") - f.Set(reflect.ValueOf(val)) - default: - iterateStruct(fTyp, f, fKey, true, nil) - } - default: - panic(fmt.Sprintf("iterateStruct: unhandled type %T", k)) - } +func initStruct(f reflect.Value, fKey string) { } func jsonName(f reflect.StructField) string { From b8da231d184ba5befa358e05e010830ee16dff58 Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 7 Sep 2020 14:05:03 +0200 Subject: [PATCH 3/6] remove IsNil --- model/modeldecoder/generator/generator.go | 12 +-- model/modeldecoder/nullable/nullable.go | 42 +++------- model/modeldecoder/nullable/nullable_test.go | 41 ++++------ model/modeldecoder/rumv3/decoder.go | 28 +++---- model/modeldecoder/rumv3/model_generated.go | 12 +-- model/modeldecoder/v2/decoder.go | 80 ++++++++++---------- model/modeldecoder/v2/model_generated.go | 16 ++-- 7 files changed, 97 insertions(+), 134 deletions(-) diff --git a/model/modeldecoder/generator/generator.go b/model/modeldecoder/generator/generator.go index c21394e84cf..d9f92e3005e 100644 --- a/model/modeldecoder/generator/generator.go +++ b/model/modeldecoder/generator/generator.go @@ -363,10 +363,10 @@ default: switch rule { case ruleRequired: fmt.Fprintf(&g.buf, ` -if !m.%s.IsSet() || m.%s.IsNil() { +if !m.%s.IsSet() { return fmt.Errorf("'%s' required") } -`[1:], f.name, f.name, flattenedName) +`[1:], f.name, flattenedName) case ruleMax: fmt.Fprintf(&g.buf, ` if utf8.RuneCountInString(m.%s.Val) > %s{ @@ -389,10 +389,10 @@ if !%s.MatchString(m.%s.Val){ switch rule { case ruleRequired: fmt.Fprintf(&g.buf, ` -if !m.%s.IsSet() || m.%s.IsNil(){ +if !m.%s.IsSet() { return fmt.Errorf("'%s' required") } -`[1:], f.name, f.name, flattenedName) +`[1:], f.name, flattenedName) case ruleMax: fmt.Fprintf(&g.buf, ` if m.%s.Val > %s{ @@ -413,10 +413,10 @@ if m.%s.Val > %s{ switch rule { case ruleRequired: fmt.Fprintf(&g.buf, ` -if !m.%s.IsSet() || m.%s.IsNil(){ +if !m.%s.IsSet() { return fmt.Errorf("'%s' required") } -`[1:], f.name, f.name, flattenedName) +`[1:], f.name, flattenedName) case ruleMax: //handled in switch statement for string types case ruleTypes: diff --git a/model/modeldecoder/nullable/nullable.go b/model/modeldecoder/nullable/nullable.go index 9d3940565bd..4fe9d0559fc 100644 --- a/model/modeldecoder/nullable/nullable.go +++ b/model/modeldecoder/nullable/nullable.go @@ -30,9 +30,8 @@ func init() { iter.ReadNil() default: (*((*String)(ptr))).Val = iter.ReadString() - (*((*String)(ptr))).isNotNil = true + (*((*String)(ptr))).isSet = true } - (*((*String)(ptr))).isSet = true }) jsoniter.RegisterTypeDecoderFunc("nullable.Int", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { switch iter.WhatIsNext() { @@ -40,9 +39,8 @@ func init() { iter.ReadNil() default: (*((*Int)(ptr))).Val = iter.ReadInt() - (*((*Int)(ptr))).isNotNil = true + (*((*Int)(ptr))).isSet = true } - (*((*Int)(ptr))).isSet = true }) jsoniter.RegisterTypeDecoderFunc("nullable.Interface", func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { switch iter.WhatIsNext() { @@ -50,22 +48,20 @@ func init() { iter.ReadNil() default: (*((*Interface)(ptr))).Val = iter.Read() - (*((*Interface)(ptr))).isNotNil = true + (*((*Interface)(ptr))).isSet = true } - (*((*Interface)(ptr))).isSet = true }) } type String struct { - Val string - isSet, isNotNil bool + Val string + isSet bool } // Set sets the value func (v *String) Set(val string) { v.Val = val v.isSet = true - v.isNotNil = true } // IsSet is true when decode was called @@ -73,29 +69,22 @@ func (v *String) IsSet() bool { return v.isSet } -// IsNil is true when no value or null value was decoded -func (v *String) IsNil() bool { - return !v.isNotNil -} - // Reset sets the String to it's initial state // where it is not set and has no value func (v *String) Reset() { v.Val = "" v.isSet = false - v.isNotNil = false } type Int struct { - Val int - isSet, isNotNil bool + Val int + isSet bool } // Set sets the value func (v *Int) Set(val int) { v.Val = val v.isSet = true - v.isNotNil = true } // IsSet is true when decode was called @@ -103,30 +92,23 @@ func (v *Int) IsSet() bool { return v.isSet } -// IsNil is true when no value or null value was decoded -func (v *Int) IsNil() bool { - return !v.isNotNil -} - // Reset sets the Int to it's initial state // where it is not set and has no value func (v *Int) Reset() { v.Val = 0 v.isSet = false - v.isNotNil = false } // TODO(simitt): follow up on https://github.com/elastic/apm-server/pull/4154#discussion_r484166721 type Interface struct { - Val interface{} `json:"val,omitempty"` - isSet, isNotNil bool + Val interface{} `json:"val,omitempty"` + isSet bool } // Set sets the value func (v *Interface) Set(val interface{}) { v.Val = val v.isSet = true - v.isNotNil = true } // IsSet is true when decode was called @@ -134,15 +116,9 @@ func (v *Interface) IsSet() bool { return v.isSet } -// IsNil is true when no value or null value was decoded -func (v *Interface) IsNil() bool { - return !v.isNotNil -} - // Reset sets the Interface to it's initial state // where it is not set and has no value func (v *Interface) Reset() { v.Val = nil v.isSet = false - v.isNotNil = false } diff --git a/model/modeldecoder/nullable/nullable_test.go b/model/modeldecoder/nullable/nullable_test.go index a89156f1005..3c27a8a938a 100644 --- a/model/modeldecoder/nullable/nullable_test.go +++ b/model/modeldecoder/nullable/nullable_test.go @@ -39,14 +39,14 @@ func TestString(t *testing.T) { name string input string - val string - isSet, isNil, fail bool + val string + isSet, fail bool }{ {name: "values", input: `{"s":"agent-go"}`, val: "agent-go", isSet: true}, {name: "empty", input: `{"s":""}`, isSet: true}, - {name: "null", input: `{"s":null}`, isSet: true, isNil: true}, - {name: "missing", input: `{}`, isNil: true}, - {name: "invalid", input: `{"s":1234}`, isNil: true, fail: true}, + {name: "null", input: `{"s":null}`}, + {name: "missing", input: `{}`}, + {name: "invalid", input: `{"s":1234}`, fail: true}, } { t.Run(tc.name, func(t *testing.T) { dec := json.NewDecoder(strings.NewReader(tc.input)) @@ -56,18 +56,15 @@ func TestString(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - assert.Equal(t, tc.isNil, testStruct.S.IsNil()) assert.Equal(t, tc.isSet, testStruct.S.IsSet()) assert.Equal(t, tc.val, testStruct.S.Val) } testStruct.S.Reset() - assert.True(t, testStruct.S.IsNil()) assert.False(t, testStruct.S.IsSet()) assert.Empty(t, testStruct.S.Val) testStruct.S.Set("teststring") - assert.False(t, testStruct.S.IsNil()) assert.True(t, testStruct.S.IsSet()) assert.Equal(t, "teststring", testStruct.S.Val) }) @@ -79,14 +76,14 @@ func TestInt(t *testing.T) { name string input string - val int - isSet, isNil, fail bool + val int + isSet, fail bool }{ {name: "values", input: `{"i":44}`, val: 44, isSet: true}, {name: "empty", input: `{"i":0}`, isSet: true}, - {name: "null", input: `{"i":null}`, isSet: true, isNil: true}, - {name: "missing", input: `{}`, isNil: true}, - {name: "invalid", input: `{"i":"1.0.1"}`, isNil: true, fail: true}, + {name: "null", input: `{"i":null}`, isSet: false}, + {name: "missing", input: `{}`}, + {name: "invalid", input: `{"i":"1.0.1"}`, fail: true}, } { t.Run(tc.name, func(t *testing.T) { dec := json.NewDecoder(strings.NewReader(tc.input)) @@ -96,18 +93,15 @@ func TestInt(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - assert.Equal(t, tc.isNil, testStruct.I.IsNil()) assert.Equal(t, tc.isSet, testStruct.I.IsSet()) assert.Equal(t, tc.val, testStruct.I.Val) } testStruct.I.Reset() - assert.True(t, testStruct.I.IsNil()) assert.False(t, testStruct.I.IsSet()) assert.Empty(t, testStruct.I.Val) testStruct.I.Set(55) - assert.False(t, testStruct.I.IsNil()) assert.True(t, testStruct.I.IsSet()) assert.Equal(t, 55, testStruct.I.Val) }) @@ -119,15 +113,15 @@ func TestInterface(t *testing.T) { name string input string - val interface{} - isSet, isNil, fail bool + val interface{} + isSet, fail bool }{ {name: "integer", input: `{"v":44}`, val: float64(44), isSet: true}, {name: "string", input: `{"v":"1.0.1"}`, val: "1.0.1", isSet: true}, {name: "bool", input: `{"v":true}`, val: true, isSet: true}, {name: "empty", input: `{"v":""}`, val: "", isSet: true}, - {name: "null", input: `{"v":null}`, isSet: true, isNil: true}, - {name: "missing", input: `{}`, isNil: true}, + {name: "null", input: `{"v":null}`}, + {name: "missing", input: `{}`}, } { t.Run(tc.name, func(t *testing.T) { dec := json.NewDecoder(strings.NewReader(tc.input)) @@ -137,20 +131,13 @@ func TestInterface(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - assert.Equal(t, tc.isNil, testStruct.V.IsNil()) assert.Equal(t, tc.isSet, testStruct.V.IsSet()) assert.Equal(t, tc.val, testStruct.V.Val) } testStruct.V.Reset() - assert.True(t, testStruct.V.IsNil()) assert.False(t, testStruct.V.IsSet()) assert.Empty(t, testStruct.V.Val) - - testStruct.V.Set("teststring") - assert.False(t, testStruct.V.IsNil()) - assert.True(t, testStruct.V.IsSet()) - assert.Equal(t, "teststring", testStruct.V.Val) }) } } diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index 9c6061acc8e..576c9106a9b 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -64,48 +64,48 @@ func mapToModel(m *metadata, out *model.Metadata) { out.Labels.Update(m.Labels) // Service - if !m.Service.Agent.Name.IsNil() { + if m.Service.Agent.Name.IsSet() { out.Service.Agent.Name = m.Service.Agent.Name.Val } - if !m.Service.Agent.Version.IsNil() { + if m.Service.Agent.Version.IsSet() { out.Service.Agent.Version = m.Service.Agent.Version.Val } - if !m.Service.Environment.IsNil() { + if m.Service.Environment.IsSet() { out.Service.Environment = m.Service.Environment.Val } - if !m.Service.Framework.Name.IsNil() { + if m.Service.Framework.Name.IsSet() { out.Service.Framework.Name = m.Service.Framework.Name.Val } - if !m.Service.Framework.Version.IsNil() { + if m.Service.Framework.Version.IsSet() { out.Service.Framework.Version = m.Service.Framework.Version.Val } - if !m.Service.Language.Name.IsNil() { + if m.Service.Language.Name.IsSet() { out.Service.Language.Name = m.Service.Language.Name.Val } - if !m.Service.Language.Version.IsNil() { + if m.Service.Language.Version.IsSet() { out.Service.Language.Version = m.Service.Language.Version.Val } - if !m.Service.Name.IsNil() { + if m.Service.Name.IsSet() { out.Service.Name = m.Service.Name.Val } - if !m.Service.Runtime.Name.IsNil() { + if m.Service.Runtime.Name.IsSet() { out.Service.Runtime.Name = m.Service.Runtime.Name.Val } - if !m.Service.Runtime.Version.IsNil() { + if m.Service.Runtime.Version.IsSet() { out.Service.Runtime.Version = m.Service.Runtime.Version.Val } - if !m.Service.Version.IsNil() { + if m.Service.Version.IsSet() { out.Service.Version = m.Service.Version.Val } // User - if !m.User.ID.IsNil() { + if m.User.ID.IsSet() { out.User.ID = fmt.Sprint(m.User.ID.Val) } - if !m.User.Email.IsNil() { + if m.User.Email.IsSet() { out.User.Email = m.User.Email.Val } - if !m.User.Name.IsNil() { + if m.User.Name.IsSet() { out.User.Name = m.User.Name.Val } } diff --git a/model/modeldecoder/rumv3/model_generated.go b/model/modeldecoder/rumv3/model_generated.go index 42050f2c69c..6371cffc651 100644 --- a/model/modeldecoder/rumv3/model_generated.go +++ b/model/modeldecoder/rumv3/model_generated.go @@ -126,7 +126,7 @@ func (m *metadataService) validate() error { if !alphaNumericExtRegex.MatchString(m.Name.Val) { return fmt.Errorf("validation rule 'pattern(alphaNumericExtRegex)' violated for 'm.se.n'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'m.se.n' required") } if err := m.Runtime.validate(); err != nil { @@ -154,13 +154,13 @@ func (m *metadataServiceAgent) validate() error { if utf8.RuneCountInString(m.Name.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.a.n'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'m.se.a.n' required") } if utf8.RuneCountInString(m.Version.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.a.ve'") } - if !m.Version.IsSet() || m.Version.IsNil() { + if !m.Version.IsSet() { return fmt.Errorf("'m.se.a.ve' required") } return nil @@ -204,7 +204,7 @@ func (m *metadataServiceLanguage) validate() error { if utf8.RuneCountInString(m.Name.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.la.n'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'m.se.la.n' required") } if utf8.RuneCountInString(m.Version.Val) > 1024 { @@ -229,13 +229,13 @@ func (m *metadataServiceRuntime) validate() error { if utf8.RuneCountInString(m.Name.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.ru.n'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'m.se.ru.n' required") } if utf8.RuneCountInString(m.Version.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'm.se.ru.ve'") } - if !m.Version.IsSet() || m.Version.IsNil() { + if !m.Version.IsSet() { return fmt.Errorf("'m.se.ru.ve' required") } return nil diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index dcb1e3be615..f2766c02daa 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -84,34 +84,34 @@ func decode(decoderFn func(m *metadataRoot) error, out *model.Metadata) error { func mapToMetadataModel(m *metadata, out *model.Metadata) { // Cloud - if !m.Cloud.Account.ID.IsNil() { + if m.Cloud.Account.ID.IsSet() { out.Cloud.AccountID = m.Cloud.Account.ID.Val } - if !m.Cloud.Account.Name.IsNil() { + if m.Cloud.Account.Name.IsSet() { out.Cloud.AccountName = m.Cloud.Account.Name.Val } - if !m.Cloud.AvailabilityZone.IsNil() { + if m.Cloud.AvailabilityZone.IsSet() { out.Cloud.AvailabilityZone = m.Cloud.AvailabilityZone.Val } - if !m.Cloud.Instance.ID.IsNil() { + if m.Cloud.Instance.ID.IsSet() { out.Cloud.InstanceID = m.Cloud.Instance.ID.Val } - if !m.Cloud.Instance.Name.IsNil() { + if m.Cloud.Instance.Name.IsSet() { out.Cloud.InstanceName = m.Cloud.Instance.Name.Val } - if !m.Cloud.Machine.Type.IsNil() { + if m.Cloud.Machine.Type.IsSet() { out.Cloud.MachineType = m.Cloud.Machine.Type.Val } - if !m.Cloud.Project.ID.IsNil() { + if m.Cloud.Project.ID.IsSet() { out.Cloud.ProjectID = m.Cloud.Project.ID.Val } - if !m.Cloud.Project.Name.IsNil() { + if m.Cloud.Project.Name.IsSet() { out.Cloud.ProjectName = m.Cloud.Project.Name.Val } - if !m.Cloud.Provider.IsNil() { + if m.Cloud.Provider.IsSet() { out.Cloud.Provider = m.Cloud.Provider.Val } - if !m.Cloud.Region.IsNil() { + if m.Cloud.Region.IsSet() { out.Cloud.Region = m.Cloud.Region.Val } @@ -125,101 +125,101 @@ func mapToMetadataModel(m *metadata, out *model.Metadata) { if len(m.Process.Argv) > 0 { out.Process.Argv = m.Process.Argv } - if !m.Process.Pid.IsNil() { + if m.Process.Pid.IsSet() { out.Process.Pid = m.Process.Pid.Val } - if !m.Process.Ppid.IsNil() { + if m.Process.Ppid.IsSet() { var pid = m.Process.Ppid.Val out.Process.Ppid = &pid } - if !m.Process.Title.IsNil() { + if m.Process.Title.IsSet() { out.Process.Title = m.Process.Title.Val } // Service - if !m.Service.Agent.EphemeralID.IsNil() { + if m.Service.Agent.EphemeralID.IsSet() { out.Service.Agent.EphemeralID = m.Service.Agent.EphemeralID.Val } - if !m.Service.Agent.Name.IsNil() { + if m.Service.Agent.Name.IsSet() { out.Service.Agent.Name = m.Service.Agent.Name.Val } - if !m.Service.Agent.Version.IsNil() { + if m.Service.Agent.Version.IsSet() { out.Service.Agent.Version = m.Service.Agent.Version.Val } - if !m.Service.Environment.IsNil() { + if m.Service.Environment.IsSet() { out.Service.Environment = m.Service.Environment.Val } - if !m.Service.Framework.Name.IsNil() { + if m.Service.Framework.Name.IsSet() { out.Service.Framework.Name = m.Service.Framework.Name.Val } - if !m.Service.Framework.Version.IsNil() { + if m.Service.Framework.Version.IsSet() { out.Service.Framework.Version = m.Service.Framework.Version.Val } - if !m.Service.Language.Name.IsNil() { + if m.Service.Language.Name.IsSet() { out.Service.Language.Name = m.Service.Language.Name.Val } - if !m.Service.Language.Version.IsNil() { + if m.Service.Language.Version.IsSet() { out.Service.Language.Version = m.Service.Language.Version.Val } - if !m.Service.Name.IsNil() { + if m.Service.Name.IsSet() { out.Service.Name = m.Service.Name.Val } - if !m.Service.Node.Name.IsNil() { + if m.Service.Node.Name.IsSet() { out.Service.Node.Name = m.Service.Node.Name.Val } - if !m.Service.Runtime.Name.IsNil() { + if m.Service.Runtime.Name.IsSet() { out.Service.Runtime.Name = m.Service.Runtime.Name.Val } - if !m.Service.Runtime.Version.IsNil() { + if m.Service.Runtime.Version.IsSet() { out.Service.Runtime.Version = m.Service.Runtime.Version.Val } - if !m.Service.Version.IsNil() { + if m.Service.Version.IsSet() { out.Service.Version = m.Service.Version.Val } // System - if !m.System.Architecture.IsNil() { + if m.System.Architecture.IsSet() { out.System.Architecture = m.System.Architecture.Val } - if !m.System.ConfiguredHostname.IsNil() { + if m.System.ConfiguredHostname.IsSet() { out.System.ConfiguredHostname = m.System.ConfiguredHostname.Val } - if !m.System.Container.ID.IsNil() { + if m.System.Container.ID.IsSet() { out.System.Container.ID = m.System.Container.ID.Val } - if !m.System.DetectedHostname.IsNil() { + if m.System.DetectedHostname.IsSet() { out.System.DetectedHostname = m.System.DetectedHostname.Val } - if m.System.ConfiguredHostname.IsNil() && m.System.DetectedHostname.IsNil() { + if !m.System.ConfiguredHostname.IsSet() && !m.System.DetectedHostname.IsSet() { out.System.DetectedHostname = m.System.HostnameDeprecated.Val } - if !m.System.IP.IsNil() { + if m.System.IP.IsSet() { out.System.IP = net.ParseIP(m.System.IP.Val) } - if !m.System.Kubernetes.Namespace.IsNil() { + if m.System.Kubernetes.Namespace.IsSet() { out.System.Kubernetes.Namespace = m.System.Kubernetes.Namespace.Val } - if !m.System.Kubernetes.Node.Name.IsNil() { + if m.System.Kubernetes.Node.Name.IsSet() { out.System.Kubernetes.NodeName = m.System.Kubernetes.Node.Name.Val } - if !m.System.Kubernetes.Pod.Name.IsNil() { + if m.System.Kubernetes.Pod.Name.IsSet() { out.System.Kubernetes.PodName = m.System.Kubernetes.Pod.Name.Val } - if !m.System.Kubernetes.Pod.UID.IsNil() { + if m.System.Kubernetes.Pod.UID.IsSet() { out.System.Kubernetes.PodUID = m.System.Kubernetes.Pod.UID.Val } - if !m.System.Platform.IsNil() { + if m.System.Platform.IsSet() { out.System.Platform = m.System.Platform.Val } // User - if !m.User.ID.IsNil() { + if m.User.ID.IsSet() { out.User.ID = fmt.Sprint(m.User.ID.Val) } - if !m.User.Email.IsNil() { + if m.User.Email.IsSet() { out.User.Email = m.User.Email.Val } - if !m.User.Name.IsNil() { + if m.User.Name.IsSet() { out.User.Name = m.User.Name.Val } } diff --git a/model/modeldecoder/v2/model_generated.go b/model/modeldecoder/v2/model_generated.go index f63197ed387..b3393a8385c 100644 --- a/model/modeldecoder/v2/model_generated.go +++ b/model/modeldecoder/v2/model_generated.go @@ -135,7 +135,7 @@ func (m *metadataCloud) validate() error { if utf8.RuneCountInString(m.Provider.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.cloud.provider'") } - if !m.Provider.IsSet() || m.Provider.IsNil() { + if !m.Provider.IsSet() { return fmt.Errorf("'metadata.cloud.provider' required") } if utf8.RuneCountInString(m.Region.Val) > 1024 { @@ -243,7 +243,7 @@ func (m *metadataProcess) validate() error { if !m.IsSet() { return nil } - if !m.Pid.IsSet() || m.Pid.IsNil() { + if !m.Pid.IsSet() { return fmt.Errorf("'metadata.process.pid' required") } if utf8.RuneCountInString(m.Title.Val) > 1024 { @@ -292,7 +292,7 @@ func (m *metadataService) validate() error { if !alphaNumericExtRegex.MatchString(m.Name.Val) { return fmt.Errorf("validation rule 'pattern(alphaNumericExtRegex)' violated for 'metadata.service.name'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'metadata.service.name' required") } if err := m.Node.validate(); err != nil { @@ -327,13 +327,13 @@ func (m *metadataServiceAgent) validate() error { if utf8.RuneCountInString(m.Name.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.agent.name'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'metadata.service.agent.name' required") } if utf8.RuneCountInString(m.Version.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.agent.version'") } - if !m.Version.IsSet() || m.Version.IsNil() { + if !m.Version.IsSet() { return fmt.Errorf("'metadata.service.agent.version' required") } return nil @@ -377,7 +377,7 @@ func (m *metadataServiceLanguage) validate() error { if utf8.RuneCountInString(m.Name.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.language.name'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'metadata.service.language.name' required") } if utf8.RuneCountInString(m.Version.Val) > 1024 { @@ -420,13 +420,13 @@ func (m *metadataServiceRuntime) validate() error { if utf8.RuneCountInString(m.Name.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.runtime.name'") } - if !m.Name.IsSet() || m.Name.IsNil() { + if !m.Name.IsSet() { return fmt.Errorf("'metadata.service.runtime.name' required") } if utf8.RuneCountInString(m.Version.Val) > 1024 { return fmt.Errorf("validation rule 'max(1024)' violated for 'metadata.service.runtime.version'") } - if !m.Version.IsSet() || m.Version.IsNil() { + if !m.Version.IsSet() { return fmt.Errorf("'metadata.service.runtime.version' required") } return nil From c9ea37279ba2dee4823bc41cce9cf18f085ca1ab Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 7 Sep 2020 21:29:11 +0200 Subject: [PATCH 4/6] Add more tests --- model/modeldecoder/generator/cmd/main.go | 2 +- .../modeldecodertest/populator.go | 161 ++++++++++++++ model/modeldecoder/rumv3/decoder.go | 36 ++-- model/modeldecoder/rumv3/decoder_test.go | 114 ++++++++++ model/modeldecoder/rumv3/model.go | 14 +- model/modeldecoder/rumv3/model_generated.go | 6 +- model/modeldecoder/rumv3/model_test.go | 197 ++++++++++++++++++ model/modeldecoder/v2/decoder.go | 10 +- model/modeldecoder/v2/decoder_test.go | 34 ++- model/modeldecoder/v2/model.go | 9 +- model/modeldecoder/v2/model_generated.go | 3 +- model/modeldecoder/v2/model_test.go | 117 ++--------- 12 files changed, 560 insertions(+), 143 deletions(-) create mode 100644 model/modeldecoder/modeldecodertest/populator.go create mode 100644 model/modeldecoder/rumv3/decoder_test.go create mode 100644 model/modeldecoder/rumv3/model_test.go diff --git a/model/modeldecoder/generator/cmd/main.go b/model/modeldecoder/generator/cmd/main.go index c29bf2aa799..e0bed4fe117 100644 --- a/model/modeldecoder/generator/cmd/main.go +++ b/model/modeldecoder/generator/cmd/main.go @@ -54,7 +54,7 @@ func genV2Models() { func genRUMV3Models() { pkg := "rumv3" - rootObjs := []string{"metadataWithKey"} + rootObjs := []string{"metadataRoot"} out := filepath.Join(modeldecoderPath, pkg, "model_generated.go") gen, err := generator.NewGenerator(importPath, pkg, typPath, rootObjs) if err != nil { diff --git a/model/modeldecoder/modeldecodertest/populator.go b/model/modeldecoder/modeldecodertest/populator.go new file mode 100644 index 00000000000..5e4b7eff6c8 --- /dev/null +++ b/model/modeldecoder/modeldecodertest/populator.go @@ -0,0 +1,161 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package modeldecodertest + +import ( + "fmt" + "net" + "reflect" + "strings" + "testing" + + "github.com/elastic/apm-server/model/modeldecoder/nullable" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func InitStructValues(val reflect.Value) { + SetStructValues(val, "initialized", 1) +} + +func SetStructValues(val reflect.Value, s string, i int) { + iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { + var newVal interface{} + switch v := f.Interface().(type) { + case map[string]interface{}: + newVal = map[string]interface{}{s: s} + case common.MapStr: + newVal = common.MapStr{s: s} + case []string: + newVal = []string{s} + case []int: + newVal = []int{i, i} + case nullable.String: + v.Set(s) + newVal = v + case nullable.Int: + v.Set(i) + newVal = v + case nullable.Interface: + v.Set(s) + newVal = v + default: + if f.Type().Kind() == reflect.Struct { + return + } + panic(fmt.Sprintf("unhandled type %T for key %s", f.Type().Kind(), key)) + } + f.Set(reflect.ValueOf(newVal)) + }) +} + +func SetZeroStructValues(val reflect.Value) { + iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { + f.Set(reflect.Zero(f.Type())) + }) +} + +func SetZeroStructValue(val reflect.Value, callback func(string)) { + iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { + original := reflect.ValueOf(f.Interface()) + defer f.Set(original) // reset original value + f.Set(reflect.Zero(f.Type())) + callback(key) + }) +} + +func AssertStructValues(t *testing.T, val reflect.Value, s string, i int) { + iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { + fVal := f.Interface() + var newVal interface{} + switch fVal.(type) { + case map[string]interface{}: + newVal = map[string]interface{}{s: s} + case common.MapStr: + newVal = common.MapStr{s: s} + case []string: + newVal = []string{s} + case []int: + newVal = []int{i, i} + case string: + newVal = s + case int: + newVal = i + case *int: + iptr := f.Interface().(*int) + fVal = *iptr + newVal = i + case net.IP: + default: + if f.Type().Kind() == reflect.Struct { + return + } + panic(fmt.Sprintf("unhandled type %T for key %s", f.Type().Kind(), key)) + } + if strings.HasPrefix(key, "UserAgent") || key == "Client.IP" || key == "System.IP" { + // these values are not set by modeldecoder + return + } + assert.Equal(t, newVal, fVal, key) + }) +} + +func iterateStruct(v reflect.Value, key string, fn func(f reflect.Value, fKey string)) { + t := v.Type() + if t.Kind() != reflect.Struct { + panic(fmt.Sprintf("iterateStruct: invalid typ %T", t.Kind())) + } + if key != "" { + key += "." + } + var fKey string + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + if !f.CanSet() { + continue + } + stf := t.Field(i) + fTyp := stf.Type + name := jsonName(stf) + if name == "" { + name = stf.Name + } + fKey = fmt.Sprintf("%s%s", key, name) + + if fTyp.Kind() == reflect.Struct { + switch f.Interface().(type) { + case nullable.String, nullable.Int, nullable.Interface: + default: + iterateStruct(f, fKey, fn) + } + } + fn(f, fKey) + } +} + +func jsonName(f reflect.StructField) string { + tag, ok := f.Tag.Lookup("json") + if !ok || tag == "-" { + return "" + } + parts := strings.Split(tag, ",") + if len(parts) == 0 { + return "" + } + return parts[0] +} diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index 576c9106a9b..9c94adab32a 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -27,41 +27,43 @@ import ( ) func init() { - metadataPool.New = func() interface{} { - return &metadata{} + metadataRootPool.New = func() interface{} { + return &metadataRoot{} } } -var metadataPool sync.Pool +var metadataRootPool sync.Pool -func fetchMetadata() *metadata { - return metadataPool.Get().(*metadata) +func fetchMetadataRoot() *metadataRoot { + return metadataRootPool.Get().(*metadataRoot) } -func releaseMetadata(m *metadata) { +func releaseMetadataRoot(m *metadataRoot) { m.Reset() - metadataPool.Put(m) + metadataRootPool.Put(m) } -// DecodeMetadata uses the given decoder to create the input models, +// DecodeNestedMetadata uses the given decoder to create the input models, // then runs the defined validations on the input models // and finally maps the values fom the input model to the given *model.Metadata instance -func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { - m := metadataWithKey{Metadata: *fetchMetadata()} - defer releaseMetadata(&m.Metadata) +func DecodeNestedMetadata(d decoder.Decoder, out *model.Metadata) error { + m := fetchMetadataRoot() + defer releaseMetadataRoot(m) if err := d.Decode(&m); err != nil { - return err + return fmt.Errorf("decode error %w", err) } if err := m.validate(); err != nil { - return err + return fmt.Errorf("validation error %w", err) } - mapToModel(&m.Metadata, out) + mapToMetadataModel(&m.Metadata, out) return nil } -func mapToModel(m *metadata, out *model.Metadata) { +func mapToMetadataModel(m *metadata, out *model.Metadata) { // Labels - out.Labels = common.MapStr{} - out.Labels.Update(m.Labels) + if len(m.Labels) > 0 { + out.Labels = common.MapStr{} + out.Labels.Update(m.Labels) + } // Service if m.Service.Agent.Name.IsSet() { diff --git a/model/modeldecoder/rumv3/decoder_test.go b/model/modeldecoder/rumv3/decoder_test.go new file mode 100644 index 00000000000..cb8f1a600ce --- /dev/null +++ b/model/modeldecoder/rumv3/decoder_test.go @@ -0,0 +1,114 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rumv3 + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/decoder" + "github.com/elastic/apm-server/model" + "github.com/elastic/apm-server/model/modeldecoder/modeldecodertest" + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestResetModelOnRelease(t *testing.T) { + inp := `{"m":{"se":{"n":"service-a"}}}` + m := fetchMetadataRoot() + require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(inp)).Decode(m)) + require.True(t, m.IsSet()) + releaseMetadataRoot(m) + assert.False(t, m.IsSet()) +} + +func TestDecodeNestedMetadata(t *testing.T) { + var testMinValidMetadata = ` + {"m":{"se":{"n":"user-service","a":{"n":"go","ve":"1.0.0"}}}}` + + decodeMetadata := func(out *model.Metadata) { + dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) + require.NoError(t, DecodeNestedMetadata(dec, out)) + } + + t.Run("decode", func(t *testing.T) { + var out model.Metadata + decodeMetadata(&out) + assert.Equal(t, model.Metadata{Service: model.Service{ + Name: "user-service", + Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) + + err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode") + }) + + t.Run("validate", func(t *testing.T) { + inp := `{}` + var out model.Metadata + err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation") + }) + +} + +func TestMappingToModel(t *testing.T) { + expected := func(s string) model.Metadata { + return model.Metadata{ + Service: model.Service{Name: s, Version: s, Environment: s, + Agent: model.Agent{Name: s, Version: s}, + Language: model.Language{Name: s, Version: s}, + Runtime: model.Runtime{Name: s, Version: s}, + Framework: model.Framework{Name: s, Version: s}}, + User: model.User{Name: s, Email: s, ID: s}, + Labels: common.MapStr{s: s}, + } + } + + // setup: + // create initialized modeldecoder and empty model metadata + // map modeldecoder to model metadata and manually set + // enhanced data that are never set by the modeldecoder + val := reflect.New(reflect.TypeOf(metadata{})) + modeldecodertest.SetStructValues(val, "init", 5000) + m := val.Interface().(*metadata) + var modelM model.Metadata + mapToMetadataModel(m, &modelM) + + // iterate through model and assert values are set + assert.Equal(t, expected("init"), modelM) + + // overwrite model metadata with specified Values + // then iterate through model and assert values are overwritten + modeldecodertest.SetStructValues(val, "overwritten", 12) + m = val.Interface().(*metadata) + mapToMetadataModel(m, &modelM) + assert.Equal(t, expected("overwritten"), modelM) + + // map an empty modeldecoder metadata to the model + // and assert values are unchanged + modeldecodertest.SetZeroStructValues(val) + m = val.Interface().(*metadata) + mapToMetadataModel(m, &modelM) + assert.Equal(t, expected("overwritten"), modelM) + +} diff --git a/model/modeldecoder/rumv3/model.go b/model/modeldecoder/rumv3/model.go index 18d7035616b..f1bf5676589 100644 --- a/model/modeldecoder/rumv3/model.go +++ b/model/modeldecoder/rumv3/model.go @@ -29,14 +29,16 @@ var ( labelsRegex = regexp.MustCompile("^[^.*\"]*$") //do not allow '.' '*' '"' ) -// metadata contain event metadata +type metadataRoot struct { + Metadata metadata `json:"m" validate:"required"` +} + type metadata struct { Labels common.MapStr `json:"l" validate:"patternKeys=labelsRegex,typesVals=string;bool;number,maxVals=1024"` Service metadataService `json:"se" validate:"required"` User metadataUser `json:"u"` } -// metadataService holds information about where the data was collected type metadataService struct { Agent metadataServiceAgent `json:"a" validate:"required"` Environment nullable.String `json:"en" validate:"max=1024"` @@ -47,25 +49,21 @@ type metadataService struct { Version nullable.String `json:"ve" validate:"max=1024"` } -//metadataServiceAgent has a version and a name type metadataServiceAgent struct { Name nullable.String `json:"n" validate:"required,max=1024"` Version nullable.String `json:"ve" validate:"required,max=1024"` } -//MetadataServiceFramework has a version and name type MetadataServiceFramework struct { Name nullable.String `json:"n" validate:"max=1024"` Version nullable.String `json:"ve" validate:"max=1024"` } -//MetadataLanguage has a version and name type metadataServiceLanguage struct { Name nullable.String `json:"n" validate:"required,max=1024"` Version nullable.String `json:"ve" validate:"max=1024"` } -//MetadataRuntime has a version and name type metadataServiceRuntime struct { Name nullable.String `json:"n" validate:"required,max=1024"` Version nullable.String `json:"ve" validate:"required,max=1024"` @@ -76,7 +74,3 @@ type metadataUser struct { Email nullable.String `json:"em" validate:"max=1024"` Name nullable.String `json:"un" validate:"max=1024"` } - -type metadataWithKey struct { - Metadata metadata `json:"m" validate:"required"` -} diff --git a/model/modeldecoder/rumv3/model_generated.go b/model/modeldecoder/rumv3/model_generated.go index 6371cffc651..b5cf96e5625 100644 --- a/model/modeldecoder/rumv3/model_generated.go +++ b/model/modeldecoder/rumv3/model_generated.go @@ -25,15 +25,15 @@ import ( "unicode/utf8" ) -func (m *metadataWithKey) IsSet() bool { +func (m *metadataRoot) IsSet() bool { return m.Metadata.IsSet() } -func (m *metadataWithKey) Reset() { +func (m *metadataRoot) Reset() { m.Metadata.Reset() } -func (m *metadataWithKey) validate() error { +func (m *metadataRoot) validate() error { if err := m.Metadata.validate(); err != nil { return err } diff --git a/model/modeldecoder/rumv3/model_test.go b/model/modeldecoder/rumv3/model_test.go new file mode 100644 index 00000000000..c1d8ef2bd7d --- /dev/null +++ b/model/modeldecoder/rumv3/model_test.go @@ -0,0 +1,197 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rumv3 + +import ( + "bytes" + "encoding/json" + "io" + "os" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/decoder" + "github.com/elastic/apm-server/model/modeldecoder/modeldecodertest" +) + +func testdata(t *testing.T) io.Reader { + r, err := os.Open("../../../testdata/intake-v3/metadata.ndjson") + require.NoError(t, err) + return r +} + +func TestIsSet(t *testing.T) { + data := `{"se":{"n":"user-service"}}` + var m metadata + require.NoError(t, decoder.NewJSONIteratorDecoder(strings.NewReader(data)).Decode(&m)) + assert.True(t, m.IsSet()) + assert.True(t, m.Service.IsSet()) + assert.True(t, m.Service.Name.IsSet()) + assert.False(t, m.Service.Language.IsSet()) +} + +func TestSetReset(t *testing.T) { + var m metadataRoot + require.NoError(t, decoder.NewJSONIteratorDecoder(testdata(t)).Decode(&m)) + require.True(t, m.IsSet()) + require.NotEmpty(t, m.Metadata.Labels) + require.True(t, m.Metadata.Service.IsSet()) + require.True(t, m.Metadata.User.IsSet()) + // call Reset and ensure initial state, except for array capacity + m.Reset() + assert.False(t, m.IsSet()) + assert.Equal(t, metadataService{}, m.Metadata.Service) + assert.Equal(t, metadataUser{}, m.Metadata.User) + assert.Empty(t, m.Metadata.Labels) +} + +func TestValidationRules(t *testing.T) { + type testcase struct { + name string + errorKey string + data string + } + + strBuilder := func(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = '⌘' + } + return string(b) + } + + testMetadata := func(t *testing.T, key string, tc testcase) { + // load data + // set testcase data for given key + var data map[string]interface{} + require.NoError(t, decoder.NewJSONIteratorDecoder(testdata(t)).Decode(&data)) + meta := data["m"].(map[string]interface{}) + var keyData map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(tc.data), &keyData)) + meta[key] = keyData + + // unmarshal data into metdata struct + var m metadata + b, err := json.Marshal(meta) + require.NoError(t, err) + require.NoError(t, decoder.NewJSONIteratorDecoder(bytes.NewReader(b)).Decode(&m)) + // run validation and checks + err = m.validate() + if tc.errorKey == "" { + assert.NoError(t, err) + } else { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errorKey) + } + } + + t.Run("user", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "id-string", data: `{"id":"user123"}`}, + {name: "id-int", data: `{"id":44}`}, + {name: "id-float", errorKey: "types", data: `{"id":45.6}`}, + {name: "id-bool", errorKey: "types", data: `{"id":true}`}, + {name: "id-string-max-len", data: `{"id":"` + strBuilder(1024) + `"}`}, + {name: "id-string-max-len", errorKey: "max", data: `{"id":"` + strBuilder(1025) + `"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + testMetadata(t, "u", tc) + }) + } + }) + + t.Run("service", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "name-valid-lower", data: `"n":"abcdefghijklmnopqrstuvwxyz"`}, + {name: "name-valid-upper", data: `"n":"ABCDEFGHIJKLMNOPQRSTUVWXYZ"`}, + {name: "name-valid-digits", data: `"n":"0123456789"`}, + {name: "name-valid-special", data: `"n":"_ -"`}, + {name: "name-asterisk", errorKey: "se.n", data: `"n":"abc*"`}, + {name: "name-dot", errorKey: "se.n", data: `"n":"abc."`}, + } { + t.Run(tc.name, func(t *testing.T) { + tc.data = `{"a":{"n":"go","ve":"1.0"},` + tc.data + `}` + testMetadata(t, "se", tc) + }) + } + }) + + t.Run("max-len", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "service-environment-max-len", data: `"en":"` + strBuilder(1024) + `"`}, + {name: "service-environment-max-len", errorKey: "max", data: `"en":"` + strBuilder(1025) + `"`}, + } { + t.Run(tc.name, func(t *testing.T) { + tc.data = `{"a":{"n":"go","ve":"1.0"},"n":"my-service",` + tc.data + `}` + testMetadata(t, "se", tc) + }) + } + }) + + t.Run("labels", func(t *testing.T) { + for _, tc := range []testcase{ + {name: "valid", data: `{"k1":"v1","k2":2.3,"k3":3,"k4":true,"k5":null}`}, + {name: "restricted-type", errorKey: "typesVals", data: `{"k1":{"k2":"v1"}}`}, + {name: "key-dot", errorKey: "patternKeys", data: `{"k.1":"v1"}`}, + {name: "key-asterisk", errorKey: "patternKeys", data: `{"k*1":"v1"}`}, + {name: "key-quotemark", errorKey: "patternKeys", data: `{"k\"1":"v1"}`}, + {name: "max-len", data: `{"k1":"` + strBuilder(1024) + `"}`}, + {name: "max-len-exceeded", errorKey: "maxVals", data: `{"k1":"` + strBuilder(1025) + `"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + testMetadata(t, "l", tc) + }) + } + }) + + t.Run("required", func(t *testing.T) { + // setup: create full metadata struct with arbitrary values set + val := reflect.New(reflect.TypeOf(metadata{})) + modeldecodertest.InitStructValues(val) + + // test vanilla struct is valid + metadata := val.Interface().(*metadata) + require.NoError(t, metadata.validate()) + + // iterate through struct, remove every key one by one + // and test that validation behaves as expected + requiredKeys := map[string]interface{}{ + "se": nil, //service + "se.a": nil, //service.agent + "se.a.n": nil, //service.agent.name + "se.a.ve": nil, //service.agent.version + "se.la.n": nil, //service.language.name + "se.ru.n": nil, //service.runtime.name + "se.ru.ve": nil, //service.runtime.version + "se.n": nil, //service.name + } + modeldecodertest.SetZeroStructValue(val, func(key string) { + err := metadata.validate() + if _, ok := requiredKeys[key]; ok { + require.Error(t, err, key) + assert.Contains(t, err.Error(), key) + } else { + assert.NoError(t, err, key) + } + }) + }) +} diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index f2766c02daa..1305be35490 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -19,7 +19,6 @@ package v2 import ( "fmt" - "net" "sync" "github.com/elastic/apm-server/decoder" @@ -31,9 +30,6 @@ func init() { metadataRootPool.New = func() interface{} { return &metadataRoot{} } - // metadataNoKeyPool.New = func() interface{} { - // return &metadataNoKey{} - // } } var metadataRootPool sync.Pool @@ -190,12 +186,10 @@ func mapToMetadataModel(m *metadata, out *model.Metadata) { if m.System.DetectedHostname.IsSet() { out.System.DetectedHostname = m.System.DetectedHostname.Val } - if !m.System.ConfiguredHostname.IsSet() && !m.System.DetectedHostname.IsSet() { + if !m.System.ConfiguredHostname.IsSet() && !m.System.DetectedHostname.IsSet() && + m.System.HostnameDeprecated.IsSet() { out.System.DetectedHostname = m.System.HostnameDeprecated.Val } - if m.System.IP.IsSet() { - out.System.IP = net.ParseIP(m.System.IP.Val) - } if m.System.Kubernetes.Namespace.IsSet() { out.System.Kubernetes.Namespace = m.System.Kubernetes.Namespace.Val } diff --git a/model/modeldecoder/v2/decoder_test.go b/model/modeldecoder/v2/decoder_test.go index 75be981af7d..cb0c1187026 100644 --- a/model/modeldecoder/v2/decoder_test.go +++ b/model/modeldecoder/v2/decoder_test.go @@ -18,6 +18,8 @@ package v2 import ( + "net" + "reflect" "strings" "testing" @@ -26,6 +28,7 @@ import ( "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" + "github.com/elastic/apm-server/model/modeldecoder/modeldecodertest" ) func TestResetModelOnRelease(t *testing.T) { @@ -158,6 +161,33 @@ func TestDecodeMetadata(t *testing.T) { } func TestMappingToModel(t *testing.T) { - //TODO(simitt) - // override existing values if they are set + // setup: + // create initialized modeldecoder and empty model metadata + // map modeldecoder to model metadata and manually set + // enhanced data that are never set by the modeldecoder + val := reflect.New(reflect.TypeOf(metadata{})) + modeldecodertest.SetStructValues(val, "init", 5000) + m := val.Interface().(*metadata) + var modelM model.Metadata + modelM.System.IP, modelM.Client.IP = net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.1") + modelM.UserAgent.Original, modelM.UserAgent.Name = "Firefox/15.0.1", "Firefox/15.0.1" + mapToMetadataModel(m, &modelM) + + // iterate through model and assert values are set + modeldecodertest.AssertStructValues(t, reflect.ValueOf(&modelM), "init", 5000) + + // overwrite model metadata with specified Values + // then iterate through model and assert values are overwritten + modeldecodertest.SetStructValues(val, "overwritten", 12) + m = val.Interface().(*metadata) + mapToMetadataModel(m, &modelM) + modeldecodertest.AssertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) + + // map an empty modeldecoder metadata to the model + // and assert values are unchanged + modeldecodertest.SetZeroStructValues(val) + m = val.Interface().(*metadata) + mapToMetadataModel(m, &modelM) + modeldecodertest.AssertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) + } diff --git a/model/modeldecoder/v2/model.go b/model/modeldecoder/v2/model.go index 86743315af0..20e35fe9948 100644 --- a/model/modeldecoder/v2/model.go +++ b/model/modeldecoder/v2/model.go @@ -29,6 +29,10 @@ var ( labelsRegex = regexp.MustCompile("^[^.*\"]*$") //do not allow '.' '*' '"' ) +type metadataRoot struct { + Metadata metadata `json:"metadata" validate:"required"` +} + type metadata struct { Cloud metadataCloud `json:"cloud"` Labels common.MapStr `json:"labels" validate:"patternKeys=labelsRegex,typesVals=string;bool;number,maxVals=1024"` @@ -116,7 +120,6 @@ type metadataSystem struct { Container metadataSystemContainer `json:"container"` DetectedHostname nullable.String `json:"detected_hostname" validate:"max=1024"` HostnameDeprecated nullable.String `json:"hostname" validate:"max=1024"` - IP nullable.String `json:"ip"` Kubernetes metadataSystemKubernetes `json:"kubernetes"` Platform nullable.String `json:"platform" validate:"max=1024"` } @@ -148,7 +151,3 @@ type metadataUser struct { Email nullable.String `json:"email" validate:"max=1024"` Name nullable.String `json:"username" validate:"max=1024"` } - -type metadataRoot struct { - Metadata metadata `json:"metadata" validate:"required"` -} diff --git a/model/modeldecoder/v2/model_generated.go b/model/modeldecoder/v2/model_generated.go index b3393a8385c..519c6155244 100644 --- a/model/modeldecoder/v2/model_generated.go +++ b/model/modeldecoder/v2/model_generated.go @@ -433,7 +433,7 @@ func (m *metadataServiceRuntime) validate() error { } func (m *metadataSystem) IsSet() bool { - return m.Architecture.IsSet() || m.ConfiguredHostname.IsSet() || m.Container.IsSet() || m.DetectedHostname.IsSet() || m.HostnameDeprecated.IsSet() || m.IP.IsSet() || m.Kubernetes.IsSet() || m.Platform.IsSet() + return m.Architecture.IsSet() || m.ConfiguredHostname.IsSet() || m.Container.IsSet() || m.DetectedHostname.IsSet() || m.HostnameDeprecated.IsSet() || m.Kubernetes.IsSet() || m.Platform.IsSet() } func (m *metadataSystem) Reset() { @@ -442,7 +442,6 @@ func (m *metadataSystem) Reset() { m.Container.Reset() m.DetectedHostname.Reset() m.HostnameDeprecated.Reset() - m.IP.Reset() m.Kubernetes.Reset() m.Platform.Reset() } diff --git a/model/modeldecoder/v2/model_test.go b/model/modeldecoder/v2/model_test.go index 7acc59821e5..9b37e889f26 100644 --- a/model/modeldecoder/v2/model_test.go +++ b/model/modeldecoder/v2/model_test.go @@ -20,7 +20,7 @@ package v2 import ( "bytes" "encoding/json" - "fmt" + "io" "os" "reflect" "strings" @@ -30,12 +30,14 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/apm-server/decoder" - "github.com/elastic/apm-server/model/modeldecoder/nullable" - "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/apm-server/model/modeldecoder/modeldecodertest" ) -var testMinValidMetadata = ` -{"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}` +func testdata(t *testing.T) io.Reader { + r, err := os.Open("../../../testdata/intake-v2/metadata.ndjson") + require.NoError(t, err) + return r +} func TestIsSet(t *testing.T) { inp := `{"cloud":{"availability_zone":"eu-west-3","instance":{"id":"1234"}}}` @@ -50,9 +52,7 @@ func TestIsSet(t *testing.T) { func TestSetReset(t *testing.T) { var m metadataRoot - r, err := os.Open("../../../testdata/intake-v2/metadata.ndjson") - require.NoError(t, err) - require.NoError(t, decoder.NewJSONIteratorDecoder(r).Decode(&m)) + require.NoError(t, decoder.NewJSONIteratorDecoder(testdata(t)).Decode(&m)) require.True(t, m.IsSet()) require.True(t, m.Metadata.Cloud.IsSet()) require.NotEmpty(t, m.Metadata.Labels) @@ -91,17 +91,19 @@ func TestValidationRules(t *testing.T) { return string(b) } - testMetadata := func(key string, tc testcase) { - // load minimal required data + testMetadata := func(t *testing.T, key string, tc testcase) { + // load data // set testcase data for given key var data map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(testMinValidMetadata), &data)) + require.NoError(t, decoder.NewJSONIteratorDecoder(testdata(t)).Decode(&data)) + meta := data["metadata"].(map[string]interface{}) var keyData map[string]interface{} require.NoError(t, json.Unmarshal([]byte(tc.data), &keyData)) - data[key] = keyData + meta[key] = keyData + // unmarshal data into metdata struct var m metadata - b, err := json.Marshal(data) + b, err := json.Marshal(meta) require.NoError(t, err) require.NoError(t, decoder.NewJSONIteratorDecoder(bytes.NewReader(b)).Decode(&m)) // run validation and checks @@ -124,7 +126,7 @@ func TestValidationRules(t *testing.T) { {name: "id-string-max-len", errorKey: "max", data: `{"id":"` + strBuilder(1025) + `"}`}, } { t.Run(tc.name, func(t *testing.T) { - testMetadata("user", tc) + testMetadata(t, "user", tc) }) } }) @@ -140,7 +142,7 @@ func TestValidationRules(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { tc.data = `{"agent":{"name":"go","version":"1.0"},` + tc.data + `}` - testMetadata("service", tc) + testMetadata(t, "service", tc) }) } }) @@ -156,7 +158,7 @@ func TestValidationRules(t *testing.T) { {name: "max-len-exceeded", errorKey: "maxVals", data: `{"k1":"` + strBuilder(1025) + `"}`}, } { t.Run(tc.name, func(t *testing.T) { - testMetadata("labels", tc) + testMetadata(t, "labels", tc) }) } }) @@ -169,43 +171,15 @@ func TestValidationRules(t *testing.T) { data: `{"pid":1,"title":"` + strBuilder(1025) + `"}`}, } { t.Run(tc.name, func(t *testing.T) { - testMetadata("process", tc) + testMetadata(t, "process", tc) }) } }) t.Run("required", func(t *testing.T) { // setup: create full metadata struct with arbitrary values set - typ := reflect.TypeOf(metadata{}) - val := reflect.New(typ) - iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { - var newVal interface{} - switch val := f.Interface().(type) { - case map[string]interface{}: - newVal = map[string]interface{}{"k1": "v1"} - case common.MapStr: - newVal = common.MapStr{"k1": "v1"} - case []string: - newVal = []string{"a", "b"} - case []int: - newVal = []int{1, 2, 3} - case nullable.String: - val.Set("teststring") - newVal = val - case nullable.Int: - val.Set(123) - newVal = val - case nullable.Interface: - val.Set("testinterface") - newVal = val - default: - if f.Type().Kind() == reflect.Struct { - return - } - panic(fmt.Sprintf("initStruct: unhandled type %T", f.Type().Kind())) - } - f.Set(reflect.ValueOf(newVal)) - }) + val := reflect.New(reflect.TypeOf(metadata{})) + modeldecodertest.InitStructValues(val) // test vanilla struct is valid metadata := val.Interface().(*metadata) @@ -225,10 +199,7 @@ func TestValidationRules(t *testing.T) { "service.runtime.version": nil, "service.name": nil, } - iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { - original := reflect.ValueOf(f.Interface()) - defer f.Set(original) // reset original value - f.Set(reflect.Zero(f.Type())) + modeldecodertest.SetZeroStructValue(val, func(key string) { err := metadata.validate() if _, ok := requiredKeys[key]; ok { require.Error(t, err, key) @@ -239,47 +210,3 @@ func TestValidationRules(t *testing.T) { }) }) } - -func iterateStruct(v reflect.Value, key string, cb func(f reflect.Value, fKey string)) { - t := v.Type() - if t.Kind() != reflect.Struct { - panic(fmt.Sprintf("iterateStruct: invalid typ %T", t.Kind())) - } - if key != "" { - key += "." - } - var fKey string - for i := 0; i < v.NumField(); i++ { - f := v.Field(i) - if !f.CanSet() { - continue - } - stf := t.Field(i) - fTyp := stf.Type - fKey = fmt.Sprintf("%s%s", key, jsonName(stf)) - - if fTyp.Kind() == reflect.Struct { - switch f.Interface().(type) { - case nullable.String, nullable.Int, nullable.Interface: - default: - iterateStruct(f, fKey, cb) - } - } - cb(f, fKey) - } -} - -func initStruct(f reflect.Value, fKey string) { -} - -func jsonName(f reflect.StructField) string { - tag, ok := f.Tag.Lookup("json") - if !ok || tag == "-" { - return "" - } - parts := strings.Split(tag, ",") - if len(parts) == 0 { - return "" - } - return parts[0] -} From 7f8a7d59ccfb9aa96b8d094e356e116763d1d826 Mon Sep 17 00:00:00 2001 From: simitt Date: Tue, 8 Sep 2020 10:34:00 +0200 Subject: [PATCH 5/6] more cleanup, PR comments --- model/modeldecoder/generator/cmd/main.go | 9 +- .../modeldecodertest/populator.go | 52 ++--- model/modeldecoder/nullable/nullable.go | 7 + model/modeldecoder/rumv3/decoder.go | 12 +- model/modeldecoder/rumv3/decoder_test.go | 17 +- model/modeldecoder/rumv3/model.go | 3 +- model/modeldecoder/v2/decoder.go | 32 +-- model/modeldecoder/v2/decoder_test.go | 188 +++++++----------- model/modeldecoder/v2/model.go | 3 +- processor/stream/benchmark_test.go | 9 +- 10 files changed, 136 insertions(+), 196 deletions(-) diff --git a/model/modeldecoder/generator/cmd/main.go b/model/modeldecoder/generator/cmd/main.go index e0bed4fe117..2068feb12e8 100644 --- a/model/modeldecoder/generator/cmd/main.go +++ b/model/modeldecoder/generator/cmd/main.go @@ -21,6 +21,7 @@ import ( "bytes" "go/format" "os" + "path" "path/filepath" "github.com/elastic/apm-server/model/modeldecoder/generator" @@ -32,8 +33,8 @@ const ( ) var ( - importPath = filepath.Join(basePath, modeldecoderPath) - typPath = filepath.Join(importPath, "nullable") + importPath = path.Join(basePath, modeldecoderPath) + typPath = path.Join(importPath, "nullable") ) func main() { @@ -44,7 +45,7 @@ func main() { func genV2Models() { pkg := "v2" rootObjs := []string{"metadataRoot"} - out := filepath.Join(modeldecoderPath, pkg, "model_generated.go") + out := filepath.Join(filepath.FromSlash(modeldecoderPath), pkg, "model_generated.go") gen, err := generator.NewGenerator(importPath, pkg, typPath, rootObjs) if err != nil { panic(err) @@ -55,7 +56,7 @@ func genV2Models() { func genRUMV3Models() { pkg := "rumv3" rootObjs := []string{"metadataRoot"} - out := filepath.Join(modeldecoderPath, pkg, "model_generated.go") + out := filepath.Join(filepath.FromSlash(modeldecoderPath), pkg, "model_generated.go") gen, err := generator.NewGenerator(importPath, pkg, typPath, rootObjs) if err != nil { panic(err) diff --git a/model/modeldecoder/modeldecodertest/populator.go b/model/modeldecoder/modeldecodertest/populator.go index 5e4b7eff6c8..8a4bebd820f 100644 --- a/model/modeldecoder/modeldecodertest/populator.go +++ b/model/modeldecoder/modeldecodertest/populator.go @@ -19,20 +19,23 @@ package modeldecodertest import ( "fmt" - "net" "reflect" "strings" - "testing" "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/beats/v7/libbeat/common" - "github.com/stretchr/testify/assert" ) +// InitStructValues iterates through the struct fields represented by +// the given reflect.Value and initializes all fields with +// some arbitrary value. func InitStructValues(val reflect.Value) { SetStructValues(val, "initialized", 1) } +// SetStructValues iterates through the struct fields represented by +// the given reflect.Value and initializes all fields with +// the given values for strings and integers. func SetStructValues(val reflect.Value, s string, i int) { iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { var newVal interface{} @@ -64,12 +67,17 @@ func SetStructValues(val reflect.Value, s string, i int) { }) } +// SetZeroStructValues iterates through the struct fields represented by +// the given reflect.Value and sets all fields to their zero values. func SetZeroStructValues(val reflect.Value) { iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { f.Set(reflect.Zero(f.Type())) }) } +// SetZeroStructValue iterates through the struct fields represented by +// the given reflect.Value, sets a field to its zero value, +// calls the callback function and resets the field to its original value func SetZeroStructValue(val reflect.Value, callback func(string)) { iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { original := reflect.ValueOf(f.Interface()) @@ -79,40 +87,10 @@ func SetZeroStructValue(val reflect.Value, callback func(string)) { }) } -func AssertStructValues(t *testing.T, val reflect.Value, s string, i int) { - iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { - fVal := f.Interface() - var newVal interface{} - switch fVal.(type) { - case map[string]interface{}: - newVal = map[string]interface{}{s: s} - case common.MapStr: - newVal = common.MapStr{s: s} - case []string: - newVal = []string{s} - case []int: - newVal = []int{i, i} - case string: - newVal = s - case int: - newVal = i - case *int: - iptr := f.Interface().(*int) - fVal = *iptr - newVal = i - case net.IP: - default: - if f.Type().Kind() == reflect.Struct { - return - } - panic(fmt.Sprintf("unhandled type %T for key %s", f.Type().Kind(), key)) - } - if strings.HasPrefix(key, "UserAgent") || key == "Client.IP" || key == "System.IP" { - // these values are not set by modeldecoder - return - } - assert.Equal(t, newVal, fVal, key) - }) +// IterateStruct iterates through the struct fields represented by +// the given reflect.Value and calls the given function on every field. +func IterateStruct(val reflect.Value, fn func(reflect.Value, string)) { + iterateStruct(val.Elem(), "", fn) } func iterateStruct(v reflect.Value, key string, fn func(f reflect.Value, fKey string)) { diff --git a/model/modeldecoder/nullable/nullable.go b/model/modeldecoder/nullable/nullable.go index 4fe9d0559fc..b4c5daad3aa 100644 --- a/model/modeldecoder/nullable/nullable.go +++ b/model/modeldecoder/nullable/nullable.go @@ -53,6 +53,8 @@ func init() { }) } +// String stores a string value and the +// information if the value has been set type String struct { Val string isSet bool @@ -76,6 +78,8 @@ func (v *String) Reset() { v.isSet = false } +// Int stores an int value and the +// information if the value has been set type Int struct { Val int isSet bool @@ -99,6 +103,9 @@ func (v *Int) Reset() { v.isSet = false } +// Interface stores an interface{} value and the +// information if the value has been set +// // TODO(simitt): follow up on https://github.com/elastic/apm-server/pull/4154#discussion_r484166721 type Interface struct { Val interface{} `json:"val,omitempty"` diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index 9c94adab32a..440d5c66eb4 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -21,22 +21,22 @@ import ( "fmt" "sync" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" - "github.com/elastic/beats/v7/libbeat/common" ) -func init() { - metadataRootPool.New = func() interface{} { +var metadataRootPool = sync.Pool{ + New: func() interface{} { return &metadataRoot{} - } + }, } -var metadataRootPool sync.Pool - func fetchMetadataRoot() *metadataRoot { return metadataRootPool.Get().(*metadataRoot) } + func releaseMetadataRoot(m *metadataRoot) { m.Reset() metadataRootPool.Put(m) diff --git a/model/modeldecoder/rumv3/decoder_test.go b/model/modeldecoder/rumv3/decoder_test.go index cb8f1a600ce..ce57c7259a2 100644 --- a/model/modeldecoder/rumv3/decoder_test.go +++ b/model/modeldecoder/rumv3/decoder_test.go @@ -25,10 +25,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/modeldecoder/modeldecodertest" - "github.com/elastic/beats/v7/libbeat/common" ) func TestResetModelOnRelease(t *testing.T) { @@ -41,19 +42,13 @@ func TestResetModelOnRelease(t *testing.T) { } func TestDecodeNestedMetadata(t *testing.T) { - var testMinValidMetadata = ` - {"m":{"se":{"n":"user-service","a":{"n":"go","ve":"1.0.0"}}}}` - - decodeMetadata := func(out *model.Metadata) { - dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) - require.NoError(t, DecodeNestedMetadata(dec, out)) - } - t.Run("decode", func(t *testing.T) { var out model.Metadata - decodeMetadata(&out) + testMinValidMetadata := `{"m":{"se":{"n":"name","a":{"n":"go","ve":"1.0.0"}}}}` + dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) + require.NoError(t, DecodeNestedMetadata(dec, &out)) assert.Equal(t, model.Metadata{Service: model.Service{ - Name: "user-service", + Name: "name", Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) diff --git a/model/modeldecoder/rumv3/model.go b/model/modeldecoder/rumv3/model.go index f1bf5676589..346a10204f3 100644 --- a/model/modeldecoder/rumv3/model.go +++ b/model/modeldecoder/rumv3/model.go @@ -20,8 +20,9 @@ package rumv3 import ( "regexp" - "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/beats/v7/libbeat/common" + + "github.com/elastic/apm-server/model/modeldecoder/nullable" ) var ( diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index 1305be35490..bb2b95ec42a 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -21,22 +21,22 @@ import ( "fmt" "sync" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" - "github.com/elastic/beats/v7/libbeat/common" ) -func init() { - metadataRootPool.New = func() interface{} { +var metadataRootPool = sync.Pool{ + New: func() interface{} { return &metadataRoot{} - } + }, } -var metadataRootPool sync.Pool - func fetchMetadataRoot() *metadataRoot { return metadataRootPool.Get().(*metadataRoot) } + func releaseMetadataRoot(m *metadataRoot) { m.Reset() metadataRootPool.Put(m) @@ -49,9 +49,7 @@ func releaseMetadataRoot(m *metadataRoot) { // DecodeMetadata should be used when the underlying byte stream does not contain the // `metadata` key, but only the metadata. func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { - return decode(func(m *metadataRoot) error { - return d.Decode(&m.Metadata) - }, out) + return decode(decodeIntoMetadata, d, out) } // DecodeNestedMetadata uses the given decoder to create the input models, @@ -60,15 +58,13 @@ func DecodeMetadata(d decoder.Decoder, out *model.Metadata) error { // // DecodeNestedMetadata should be used when the underlying byte stream does start with the `metadata` key func DecodeNestedMetadata(d decoder.Decoder, out *model.Metadata) error { - return decode(func(m *metadataRoot) error { - return d.Decode(m) - }, out) + return decode(decodeIntoMetadataRoot, d, out) } -func decode(decoderFn func(m *metadataRoot) error, out *model.Metadata) error { +func decode(decFn func(d decoder.Decoder, m *metadataRoot) error, d decoder.Decoder, out *model.Metadata) error { m := fetchMetadataRoot() defer releaseMetadataRoot(m) - if err := decoderFn(m); err != nil { + if err := decFn(d, m); err != nil { return fmt.Errorf("decode error %w", err) } if err := m.validate(); err != nil { @@ -78,6 +74,14 @@ func decode(decoderFn func(m *metadataRoot) error, out *model.Metadata) error { return nil } +func decodeIntoMetadata(d decoder.Decoder, m *metadataRoot) error { + return d.Decode(&m.Metadata) +} + +func decodeIntoMetadataRoot(d decoder.Decoder, m *metadataRoot) error { + return d.Decode(m) +} + func mapToMetadataModel(m *metadata, out *model.Metadata) { // Cloud if m.Cloud.Account.ID.IsSet() { diff --git a/model/modeldecoder/v2/decoder_test.go b/model/modeldecoder/v2/decoder_test.go index cb0c1187026..0561a4d739b 100644 --- a/model/modeldecoder/v2/decoder_test.go +++ b/model/modeldecoder/v2/decoder_test.go @@ -18,6 +18,7 @@ package v2 import ( + "fmt" "net" "reflect" "strings" @@ -26,6 +27,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/modeldecoder/modeldecodertest" @@ -40,124 +43,40 @@ func TestResetModelOnRelease(t *testing.T) { assert.False(t, m.IsSet()) } -func TestDecodeProfileMetadata(t *testing.T) { - var testMinValidMetadata = ` - {"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}` - - decodeProfileMetadata := func(out *model.Metadata) { - dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) - require.NoError(t, DecodeMetadata(dec, out)) - } - - t.Run("fetch-release", func(t *testing.T) { - // this test cannot be run in parallel with other tests using the - // metadataNoKeyPool sync.Pool - - // whenever DecodeProfileMetadata is called a metadata instance - // should be retrieved from the sync.Pool and be released on finish - // test behavior by overriding the new method, occupying an existing - // instance and counting how often the New method is called - var newCount, expectedNewCount int - origNew := metadataRootPool.New - defer func() { metadataRootPool.New = origNew }() - metadataRootPool.New = func() interface{} { - newCount++ - return &metadataRoot{} - } - var out model.Metadata - // on the first call align the expected with the current new count - // important since other tests might have run already - decodeProfileMetadata(&out) - expectedNewCount = newCount - // on the second call it should reuse the metadata instance - decodeProfileMetadata(&out) - assert.Equal(t, expectedNewCount, newCount) - // force a new instance on the next decoder call - fetchMetadataRoot() - decodeProfileMetadata(&out) - assert.Equal(t, expectedNewCount+1, newCount) - }) - - t.Run("decode", func(t *testing.T) { - var out model.Metadata - decodeProfileMetadata(&out) - assert.Equal(t, model.Metadata{Service: model.Service{ - Name: "user-service", - Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) - - err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) - require.Error(t, err) - assert.Contains(t, err.Error(), "decode") - }) - - t.Run("validate", func(t *testing.T) { - inp := `{}` - var out model.Metadata - err := DecodeMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) - require.Error(t, err) - assert.Contains(t, err.Error(), "validation") - }) - -} - func TestDecodeMetadata(t *testing.T) { - var testMinValidMetadata = ` - {"metadata":{"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}}` - decodeMetadata := func(out *model.Metadata) { - dec := decoder.NewJSONIteratorDecoder(strings.NewReader(testMinValidMetadata)) - require.NoError(t, DecodeNestedMetadata(dec, out)) + for _, tc := range []struct { + name string + input string + decodeFn func(decoder.Decoder, *model.Metadata) error + }{ + {name: "decodeMetadata", decodeFn: DecodeMetadata, + input: `{"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}`}, + {name: "decodeNestedMetadata", decodeFn: DecodeNestedMetadata, + input: `{"metadata":{"service":{"name":"user-service","agent":{"name":"go","version":"1.0.0"}}}}`}, + } { + t.Run("decode", func(t *testing.T) { + var out model.Metadata + dec := decoder.NewJSONIteratorDecoder(strings.NewReader(tc.input)) + require.NoError(t, tc.decodeFn(dec, &out)) + assert.Equal(t, model.Metadata{Service: model.Service{ + Name: "user-service", + Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) + + err := tc.decodeFn(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode") + }) + + t.Run("validate", func(t *testing.T) { + inp := `{}` + var out model.Metadata + err := tc.decodeFn(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation") + }) } - t.Run("fetch-release", func(t *testing.T) { - // this test cannot be run in parallel with other tests using the - // metadataNoKeyPool sync.Pool - - // whenever DecodeProfileMetadata is called a metadata instance - // should be retrieved from the sync.Pool and be released on finish - // test behavior by overriding the new method, occupying an existing - // instance and counting how often the New method is called - var newCount, expectedNewCount int - origNew := metadataRootPool.New - defer func() { metadataRootPool.New = origNew }() - metadataRootPool.New = func() interface{} { - newCount++ - return &metadataRoot{} - } - var out model.Metadata - // on the first call align the expected with the current new count - // important since other tests might have run already - decodeMetadata(&out) - expectedNewCount = newCount - // on the second call it should reuse the metadata instance - decodeMetadata(&out) - assert.Equal(t, expectedNewCount, newCount) - // force a new instance on the next decoder call - fetchMetadataRoot() - decodeMetadata(&out) - assert.Equal(t, expectedNewCount+1, newCount) - }) - - t.Run("decode", func(t *testing.T) { - var out model.Metadata - decodeMetadata(&out) - assert.Equal(t, model.Metadata{Service: model.Service{ - Name: "user-service", - Agent: model.Agent{Name: "go", Version: "1.0.0"}}}, out) - - err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(`malformed`)), &out) - require.Error(t, err) - assert.Contains(t, err.Error(), "decode") - }) - - t.Run("validate", func(t *testing.T) { - inp := `{}` - var out model.Metadata - err := DecodeNestedMetadata(decoder.NewJSONIteratorDecoder(strings.NewReader(inp)), &out) - require.Error(t, err) - assert.Contains(t, err.Error(), "validation") - }) - } func TestMappingToModel(t *testing.T) { @@ -174,20 +93,55 @@ func TestMappingToModel(t *testing.T) { mapToMetadataModel(m, &modelM) // iterate through model and assert values are set - modeldecodertest.AssertStructValues(t, reflect.ValueOf(&modelM), "init", 5000) + assertStructValues(t, reflect.ValueOf(&modelM), "init", 5000) // overwrite model metadata with specified Values // then iterate through model and assert values are overwritten modeldecodertest.SetStructValues(val, "overwritten", 12) m = val.Interface().(*metadata) mapToMetadataModel(m, &modelM) - modeldecodertest.AssertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) + assertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) // map an empty modeldecoder metadata to the model // and assert values are unchanged modeldecodertest.SetZeroStructValues(val) m = val.Interface().(*metadata) mapToMetadataModel(m, &modelM) - modeldecodertest.AssertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) + assertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) +} +func assertStructValues(t *testing.T, val reflect.Value, s string, i int) { + modeldecodertest.IterateStruct(val, func(f reflect.Value, key string) { + fVal := f.Interface() + var newVal interface{} + switch fVal.(type) { + case map[string]interface{}: + newVal = map[string]interface{}{s: s} + case common.MapStr: + newVal = common.MapStr{s: s} + case []string: + newVal = []string{s} + case []int: + newVal = []int{i, i} + case string: + newVal = s + case int: + newVal = i + case *int: + iptr := f.Interface().(*int) + fVal = *iptr + newVal = i + case net.IP: + default: + if f.Type().Kind() == reflect.Struct { + return + } + panic(fmt.Sprintf("unhandled type %T for key %s", f.Type().Kind(), key)) + } + if strings.HasPrefix(key, "UserAgent") || key == "Client.IP" || key == "System.IP" { + // these values are not set by modeldecoder + return + } + assert.Equal(t, newVal, fVal, key) + }) } diff --git a/model/modeldecoder/v2/model.go b/model/modeldecoder/v2/model.go index 20e35fe9948..a8003993bac 100644 --- a/model/modeldecoder/v2/model.go +++ b/model/modeldecoder/v2/model.go @@ -20,8 +20,9 @@ package v2 import ( "regexp" - "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/beats/v7/libbeat/common" + + "github.com/elastic/apm-server/model/modeldecoder/nullable" ) var ( diff --git a/processor/stream/benchmark_test.go b/processor/stream/benchmark_test.go index 865bcae8a0c..d6436e22377 100644 --- a/processor/stream/benchmark_test.go +++ b/processor/stream/benchmark_test.go @@ -21,7 +21,6 @@ import ( "bytes" "context" "io/ioutil" - "math" "path/filepath" "testing" @@ -34,13 +33,13 @@ import ( func BenchmarkBackendProcessor(b *testing.B) { processor := BackendProcessor(&config.Config{MaxEventSize: 300 * 1024}) - files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v2/*.ndjson")) + files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v2/metadata.ndjson")) benchmarkStreamProcessor(b, processor, files) } func BenchmarkRUMV3Processor(b *testing.B) { processor := RUMV3Processor(&config.Config{MaxEventSize: 300 * 1024}) - files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v3/rum_*.ndjson")) + files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v3/metadata.ndjson")) benchmarkStreamProcessor(b, processor, files) } @@ -49,7 +48,7 @@ func benchmarkStreamProcessor(b *testing.B, processor *Processor, files []string return nil } //ensure to not hit rate limit as blocking wait would be measured otherwise - rl := rate.NewLimiter(rate.Limit(math.MaxFloat64-1), math.MaxInt32) + // rl := rate.NewLimiter(rate.Limit(math.MaxFloat64-1), math.MaxInt32) benchmark := func(filename string, rl *rate.Limiter) func(b *testing.B) { return func(b *testing.B) { @@ -73,7 +72,7 @@ func benchmarkStreamProcessor(b *testing.B, processor *Processor, files []string for _, f := range files { b.Run(filepath.Base(f), func(b *testing.B) { b.Run("NoRateLimit", benchmark(f, nil)) - b.Run("WithRateLimit", benchmark(f, rl)) + // b.Run("WithRateLimit", benchmark(f, rl)) }) } } From d4fdf37341415fa987925f4a206a366f87d8b872 Mon Sep 17 00:00:00 2001 From: simitt Date: Tue, 8 Sep 2020 13:46:48 +0200 Subject: [PATCH 6/6] cleanup tester API --- .../modeldecodertest/populator.go | 36 ++++++++-------- model/modeldecoder/rumv3/decoder_test.go | 19 ++++----- model/modeldecoder/rumv3/model_test.go | 8 ++-- model/modeldecoder/v2/decoder_test.go | 41 +++++++++---------- model/modeldecoder/v2/model_test.go | 9 ++-- processor/stream/benchmark_test.go | 9 ++-- 6 files changed, 57 insertions(+), 65 deletions(-) diff --git a/model/modeldecoder/modeldecodertest/populator.go b/model/modeldecoder/modeldecodertest/populator.go index 8a4bebd820f..b49fa864174 100644 --- a/model/modeldecoder/modeldecodertest/populator.go +++ b/model/modeldecoder/modeldecodertest/populator.go @@ -29,33 +29,33 @@ import ( // InitStructValues iterates through the struct fields represented by // the given reflect.Value and initializes all fields with // some arbitrary value. -func InitStructValues(val reflect.Value) { - SetStructValues(val, "initialized", 1) +func InitStructValues(i interface{}) { + SetStructValues(i, "initialized", 1) } // SetStructValues iterates through the struct fields represented by // the given reflect.Value and initializes all fields with // the given values for strings and integers. -func SetStructValues(val reflect.Value, s string, i int) { - iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { +func SetStructValues(in interface{}, vStr string, vInt int) { + IterateStruct(in, func(f reflect.Value, key string) { var newVal interface{} switch v := f.Interface().(type) { case map[string]interface{}: - newVal = map[string]interface{}{s: s} + newVal = map[string]interface{}{vStr: vStr} case common.MapStr: - newVal = common.MapStr{s: s} + newVal = common.MapStr{vStr: vStr} case []string: - newVal = []string{s} + newVal = []string{vStr} case []int: - newVal = []int{i, i} + newVal = []int{vInt, vInt} case nullable.String: - v.Set(s) + v.Set(vStr) newVal = v case nullable.Int: - v.Set(i) + v.Set(vInt) newVal = v case nullable.Interface: - v.Set(s) + v.Set(vStr) newVal = v default: if f.Type().Kind() == reflect.Struct { @@ -69,8 +69,8 @@ func SetStructValues(val reflect.Value, s string, i int) { // SetZeroStructValues iterates through the struct fields represented by // the given reflect.Value and sets all fields to their zero values. -func SetZeroStructValues(val reflect.Value) { - iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { +func SetZeroStructValues(i interface{}) { + IterateStruct(i, func(f reflect.Value, key string) { f.Set(reflect.Zero(f.Type())) }) } @@ -78,8 +78,8 @@ func SetZeroStructValues(val reflect.Value) { // SetZeroStructValue iterates through the struct fields represented by // the given reflect.Value, sets a field to its zero value, // calls the callback function and resets the field to its original value -func SetZeroStructValue(val reflect.Value, callback func(string)) { - iterateStruct(val.Elem(), "", func(f reflect.Value, key string) { +func SetZeroStructValue(i interface{}, callback func(string)) { + IterateStruct(i, func(f reflect.Value, key string) { original := reflect.ValueOf(f.Interface()) defer f.Set(original) // reset original value f.Set(reflect.Zero(f.Type())) @@ -89,7 +89,11 @@ func SetZeroStructValue(val reflect.Value, callback func(string)) { // IterateStruct iterates through the struct fields represented by // the given reflect.Value and calls the given function on every field. -func IterateStruct(val reflect.Value, fn func(reflect.Value, string)) { +func IterateStruct(i interface{}, fn func(reflect.Value, string)) { + val := reflect.ValueOf(i) + if val.Kind() != reflect.Ptr { + panic("expected pointer to struct as parameter") + } iterateStruct(val.Elem(), "", fn) } diff --git a/model/modeldecoder/rumv3/decoder_test.go b/model/modeldecoder/rumv3/decoder_test.go index ce57c7259a2..a9384334396 100644 --- a/model/modeldecoder/rumv3/decoder_test.go +++ b/model/modeldecoder/rumv3/decoder_test.go @@ -18,7 +18,6 @@ package rumv3 import ( - "reflect" "strings" "testing" @@ -83,27 +82,23 @@ func TestMappingToModel(t *testing.T) { // create initialized modeldecoder and empty model metadata // map modeldecoder to model metadata and manually set // enhanced data that are never set by the modeldecoder - val := reflect.New(reflect.TypeOf(metadata{})) - modeldecodertest.SetStructValues(val, "init", 5000) - m := val.Interface().(*metadata) + var m metadata + modeldecodertest.SetStructValues(&m, "init", 5000) var modelM model.Metadata - mapToMetadataModel(m, &modelM) - + mapToMetadataModel(&m, &modelM) // iterate through model and assert values are set assert.Equal(t, expected("init"), modelM) // overwrite model metadata with specified Values // then iterate through model and assert values are overwritten - modeldecodertest.SetStructValues(val, "overwritten", 12) - m = val.Interface().(*metadata) - mapToMetadataModel(m, &modelM) + modeldecodertest.SetStructValues(&m, "overwritten", 12) + mapToMetadataModel(&m, &modelM) assert.Equal(t, expected("overwritten"), modelM) // map an empty modeldecoder metadata to the model // and assert values are unchanged - modeldecodertest.SetZeroStructValues(val) - m = val.Interface().(*metadata) - mapToMetadataModel(m, &modelM) + modeldecodertest.SetZeroStructValues(&m) + mapToMetadataModel(&m, &modelM) assert.Equal(t, expected("overwritten"), modelM) } diff --git a/model/modeldecoder/rumv3/model_test.go b/model/modeldecoder/rumv3/model_test.go index c1d8ef2bd7d..b04866494ce 100644 --- a/model/modeldecoder/rumv3/model_test.go +++ b/model/modeldecoder/rumv3/model_test.go @@ -22,7 +22,6 @@ import ( "encoding/json" "io" "os" - "reflect" "strings" "testing" @@ -165,11 +164,10 @@ func TestValidationRules(t *testing.T) { t.Run("required", func(t *testing.T) { // setup: create full metadata struct with arbitrary values set - val := reflect.New(reflect.TypeOf(metadata{})) - modeldecodertest.InitStructValues(val) + var metadata metadata + modeldecodertest.InitStructValues(&metadata) // test vanilla struct is valid - metadata := val.Interface().(*metadata) require.NoError(t, metadata.validate()) // iterate through struct, remove every key one by one @@ -184,7 +182,7 @@ func TestValidationRules(t *testing.T) { "se.ru.ve": nil, //service.runtime.version "se.n": nil, //service.name } - modeldecodertest.SetZeroStructValue(val, func(key string) { + modeldecodertest.SetZeroStructValue(&metadata, func(key string) { err := metadata.validate() if _, ok := requiredKeys[key]; ok { require.Error(t, err, key) diff --git a/model/modeldecoder/v2/decoder_test.go b/model/modeldecoder/v2/decoder_test.go index 0561a4d739b..6cefecfc8d7 100644 --- a/model/modeldecoder/v2/decoder_test.go +++ b/model/modeldecoder/v2/decoder_test.go @@ -84,53 +84,50 @@ func TestMappingToModel(t *testing.T) { // create initialized modeldecoder and empty model metadata // map modeldecoder to model metadata and manually set // enhanced data that are never set by the modeldecoder - val := reflect.New(reflect.TypeOf(metadata{})) - modeldecodertest.SetStructValues(val, "init", 5000) - m := val.Interface().(*metadata) + var m metadata + modeldecodertest.SetStructValues(&m, "init", 5000) var modelM model.Metadata modelM.System.IP, modelM.Client.IP = net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.1") modelM.UserAgent.Original, modelM.UserAgent.Name = "Firefox/15.0.1", "Firefox/15.0.1" - mapToMetadataModel(m, &modelM) + mapToMetadataModel(&m, &modelM) // iterate through model and assert values are set - assertStructValues(t, reflect.ValueOf(&modelM), "init", 5000) + assertStructValues(t, &modelM, "init", 5000) // overwrite model metadata with specified Values // then iterate through model and assert values are overwritten - modeldecodertest.SetStructValues(val, "overwritten", 12) - m = val.Interface().(*metadata) - mapToMetadataModel(m, &modelM) - assertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) + modeldecodertest.SetStructValues(&m, "overwritten", 12) + mapToMetadataModel(&m, &modelM) + assertStructValues(t, &modelM, "overwritten", 12) // map an empty modeldecoder metadata to the model // and assert values are unchanged - modeldecodertest.SetZeroStructValues(val) - m = val.Interface().(*metadata) - mapToMetadataModel(m, &modelM) - assertStructValues(t, reflect.ValueOf(&modelM), "overwritten", 12) + modeldecodertest.SetZeroStructValues(&m) + mapToMetadataModel(&m, &modelM) + assertStructValues(t, &modelM, "overwritten", 12) } -func assertStructValues(t *testing.T, val reflect.Value, s string, i int) { - modeldecodertest.IterateStruct(val, func(f reflect.Value, key string) { +func assertStructValues(t *testing.T, i interface{}, vStr string, vInt int) { + modeldecodertest.IterateStruct(i, func(f reflect.Value, key string) { fVal := f.Interface() var newVal interface{} switch fVal.(type) { case map[string]interface{}: - newVal = map[string]interface{}{s: s} + newVal = map[string]interface{}{vStr: vStr} case common.MapStr: - newVal = common.MapStr{s: s} + newVal = common.MapStr{vStr: vStr} case []string: - newVal = []string{s} + newVal = []string{vStr} case []int: - newVal = []int{i, i} + newVal = []int{vInt, vInt} case string: - newVal = s + newVal = vStr case int: - newVal = i + newVal = vInt case *int: iptr := f.Interface().(*int) fVal = *iptr - newVal = i + newVal = vInt case net.IP: default: if f.Type().Kind() == reflect.Struct { diff --git a/model/modeldecoder/v2/model_test.go b/model/modeldecoder/v2/model_test.go index 9b37e889f26..936bd9746fe 100644 --- a/model/modeldecoder/v2/model_test.go +++ b/model/modeldecoder/v2/model_test.go @@ -22,7 +22,6 @@ import ( "encoding/json" "io" "os" - "reflect" "strings" "testing" @@ -178,11 +177,9 @@ func TestValidationRules(t *testing.T) { t.Run("required", func(t *testing.T) { // setup: create full metadata struct with arbitrary values set - val := reflect.New(reflect.TypeOf(metadata{})) - modeldecodertest.InitStructValues(val) - + var metadata metadata + modeldecodertest.InitStructValues(&metadata) // test vanilla struct is valid - metadata := val.Interface().(*metadata) require.NoError(t, metadata.validate()) // iterate through struct, remove every key one by one @@ -199,7 +196,7 @@ func TestValidationRules(t *testing.T) { "service.runtime.version": nil, "service.name": nil, } - modeldecodertest.SetZeroStructValue(val, func(key string) { + modeldecodertest.SetZeroStructValue(&metadata, func(key string) { err := metadata.validate() if _, ok := requiredKeys[key]; ok { require.Error(t, err, key) diff --git a/processor/stream/benchmark_test.go b/processor/stream/benchmark_test.go index d6436e22377..865bcae8a0c 100644 --- a/processor/stream/benchmark_test.go +++ b/processor/stream/benchmark_test.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "io/ioutil" + "math" "path/filepath" "testing" @@ -33,13 +34,13 @@ import ( func BenchmarkBackendProcessor(b *testing.B) { processor := BackendProcessor(&config.Config{MaxEventSize: 300 * 1024}) - files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v2/metadata.ndjson")) + files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v2/*.ndjson")) benchmarkStreamProcessor(b, processor, files) } func BenchmarkRUMV3Processor(b *testing.B) { processor := RUMV3Processor(&config.Config{MaxEventSize: 300 * 1024}) - files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v3/metadata.ndjson")) + files, _ := filepath.Glob(filepath.FromSlash("../../testdata/intake-v3/rum_*.ndjson")) benchmarkStreamProcessor(b, processor, files) } @@ -48,7 +49,7 @@ func benchmarkStreamProcessor(b *testing.B, processor *Processor, files []string return nil } //ensure to not hit rate limit as blocking wait would be measured otherwise - // rl := rate.NewLimiter(rate.Limit(math.MaxFloat64-1), math.MaxInt32) + rl := rate.NewLimiter(rate.Limit(math.MaxFloat64-1), math.MaxInt32) benchmark := func(filename string, rl *rate.Limiter) func(b *testing.B) { return func(b *testing.B) { @@ -72,7 +73,7 @@ func benchmarkStreamProcessor(b *testing.B, processor *Processor, files []string for _, f := range files { b.Run(filepath.Base(f), func(b *testing.B) { b.Run("NoRateLimit", benchmark(f, nil)) - // b.Run("WithRateLimit", benchmark(f, rl)) + b.Run("WithRateLimit", benchmark(f, rl)) }) } }