Fix validations based on fields imported from ECS

internal/fields/dependency_manager.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -24,6 +25,7 @@ import (
 const (
 	ecsSchemaName      = "ecs"
 	gitReferencePrefix = "git@"
+	localFilePrefix    = "file://"
 
 	ecsSchemaFile = "ecs_nested.yml"
 	ecsSchemaURL  = "https://raw.githubusercontent.com/elastic/ecs/%s/generated/ecs/%s"
@@ -70,6 +72,15 @@ func loadECSFieldsSchema(dep buildmanifest.ECSDependency) ([]FieldDefinition, er
 }
 
 func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency) ([]byte, error) {
+	if strings.HasPrefix(dep.Reference, localFilePrefix) {
+		uri, err := url.Parse(dep.Reference)
+		if err != nil {
+			return nil, err
+		}
+
+		return os.ReadFile(uri.Path)
+	}
+
 	gitReference, err := asGitReference(dep.Reference)
 	if err != nil {
 		return nil, fmt.Errorf("can't process the value as Git reference: %w", err)
@@ -152,6 +163,10 @@ type InjectFieldsOptions struct {
 	// ECS fields at the top level, when they cannot be reused there.
 	DisallowReusableECSFieldsAtTopLevel bool
 
+	// IncludeValidationSettings can be set to enable the injection of settings of imported
+	// fields that are only used for validation of documents, but are not needed on built packages.
+	IncludeValidationSettings bool
+
 	root string
 }
 
@@ -182,7 +197,7 @@ func (dm *DependencyManager) injectFieldsWithOptions(defs []common.MapStr, optio
 				return nil, false, fmt.Errorf("field %s cannot be reused at top level", fieldPath)
 			}
 
-			transformed := transformImportedField(imported)
+			transformed := transformImportedField(imported, options)
 
 			// Allow overrides of everything, except the imported type, for consistency.
 			transformed.DeepUpdate(def)
@@ -295,7 +310,7 @@ func buildFieldPath(root string, field common.MapStr) string {
 	return path
 }
 
-func transformImportedField(fd FieldDefinition) common.MapStr {
+func transformImportedField(fd FieldDefinition, options InjectFieldsOptions) common.MapStr {
 	m := common.MapStr{
 		"name": fd.Name,
 		"type": fd.Type,
@@ -318,17 +333,28 @@ func transformImportedField(fd FieldDefinition) common.MapStr {
 		m["doc_values"] = *fd.DocValues
 	}
 
-	if len(fd.Normalize) > 0 {
-		m["normalize"] = fd.Normalize
-	}
-
 	if len(fd.MultiFields) > 0 {
 		var t []common.MapStr
 		for _, f := range fd.MultiFields {
-			i := transformImportedField(f)
+			i := transformImportedField(f, options)
 			t = append(t, i)
 		}
 		m.Put("multi_fields", t)
 	}
+
+	if options.IncludeValidationSettings {
+		if len(fd.Normalize) > 0 {
+			m["normalize"] = fd.Normalize
+		}
+
+		if len(fd.AllowedValues) > 0 {
+			m["allowed_values"] = fd.AllowedValues
+		}
+
+		if len(fd.ExpectedValues) > 0 {
+			m["expected_values"] = fd.ExpectedValues
+		}
+	}
+
 	return m
 }

internal/fields/dependency_manager_test.go

@@ -5,11 +5,15 @@
 package fields
 
 import (
+	"encoding/json"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-package/internal/common"
+	"github.com/elastic/elastic-package/internal/packages/buildmanifest"
 )
 
 func TestDependencyManagerInjectExternalFields(t *testing.T) {
@@ -230,6 +234,9 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) {
 					"external": "test",
 				},
 			},
+			options: InjectFieldsOptions{
+				IncludeValidationSettings: true,
+			},
 			result: []common.MapStr{
 				{
 					"name":        "host.ip",
@@ -607,3 +614,117 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) {
 		})
 	}
 }
+
+func TestDependencyManagerWithECS(t *testing.T) {
+	const ecsNestedPath8_10_0 = "./testdata/ecs_nested_v8.10.0.yml"
+	ecsPath, _ := filepath.Abs(ecsNestedPath8_10_0)
+	deps := buildmanifest.Dependencies{
+		ECS: buildmanifest.ECSDependency{
+			Reference: "file://" + ecsPath,
+		},
+	}
+	dm, err := CreateFieldDependencyManager(deps)
+	require.NoError(t, err)
+
+	cases := []struct {
+		title   string
+		defs    []common.MapStr
+		result  []common.MapStr
+		options InjectFieldsOptions
+		checkFn func(*testing.T, []common.MapStr)
+		valid   bool
+	}{
+		{
+			title: "disallowed reusable field at lop level",
+			defs: []common.MapStr{
+				{
+					"name":     "geo.city_name",
+					"external": "ecs",
+				},
+			},
+			options: InjectFieldsOptions{
+				DisallowReusableECSFieldsAtTopLevel: true,
+			},
+			valid: false,
+		},
+		{
+			title: "legacy support to reuse field at lop level",
+			defs: []common.MapStr{
+				{
+					"name":     "geo.city_name",
+					"external": "ecs",
+				},
+			},
+			options: InjectFieldsOptions{
+				DisallowReusableECSFieldsAtTopLevel: false,
+			},
+			result: []common.MapStr{
+				{
+					"name":        "geo.city_name",
+					"description": "City name.",
+					"type":        "keyword",
+				},
+			},
+			valid: true,
+		},
+		{
+			title: "allowed values are injected for validation",
+			defs: []common.MapStr{
+				{
+					"name":     "event.type",
+					"external": "ecs",
+				},
+			},
+			options: InjectFieldsOptions{
+				IncludeValidationSettings: true,
+			},
+			valid: true,
+			checkFn: func(t *testing.T, result []common.MapStr) {
+				require.Len(t, result, 1)
+				_, ok := result[0]["allowed_values"]
+				if !assert.True(t, ok) {
+					d, _ := json.MarshalIndent(result[0], "", "  ")
+					t.Logf("expected to find allowed_values in %s", string(d))
+				}
+			},
+		},
+		{
+			title: "allowed values are not injected when not intended for validation",
+			defs: []common.MapStr{
+				{
+					"name":     "event.type",
+					"external": "ecs",
+				},
+			},
+			options: InjectFieldsOptions{
+				IncludeValidationSettings: false,
+			},
+			valid: true,
+			checkFn: func(t *testing.T, result []common.MapStr) {
+				require.Len(t, result, 1)
+				_, ok := result[0]["allowed_values"]
+				assert.False(t, ok)
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			result, _, err := dm.InjectFieldsWithOptions(c.defs, c.options)
+			if !c.valid {
+				assert.Error(t, err)
+				return
+			}
+
+			assert.NoError(t, err)
+			if len(c.result) > 0 {
+				assert.EqualValues(t, c.result, result)
+			}
+			if c.checkFn != nil {
+				t.Run("checkFn", func(t *testing.T) {
+					c.checkFn(t, result)
+				})
+			}
+		})
+	}
+}