Skip to content

Commit

Permalink
Add initial support to validate the mappings in system tests (#2214)
Browse files Browse the repository at this point in the history
Add validation of the mappings when running system tests. Currently, this
validation does not take into account dynamic templates.

This process compares the mappings installed by Fleet (preview mappings)
with the ones obtained after ingesting new documents into Elasticsearch.
This new validation needs to be enabled by setting a new environment 
variable "ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD=mappings".
By default, elastic-package keeps using the same mechanism to validate that
all fields are documented.
  • Loading branch information
mrodm authored Dec 12, 2024
1 parent b9590bd commit 6def317
Show file tree
Hide file tree
Showing 13 changed files with 1,978 additions and 65 deletions.
94 changes: 42 additions & 52 deletions .buildkite/pipeline.trigger.integration.tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,9 @@ CHECK_PACKAGES_TESTS=(
test-check-packages-benchmarks
test-check-packages-with-logstash
)
for independent_agent in false true ; do
for test in "${CHECK_PACKAGES_TESTS[@]}"; do
label_suffix=""
if [[ "$independent_agent" == "false" ]]; then
label_suffix=" (stack agent)"
fi
test_name=${test#"test-check-packages-"}
echo " - label: \":go: Integration test: ${test_name}${label_suffix}\""
echo " - label: \":go: Integration test: ${test_name}\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
echo " agents:"
echo " provider: \"gcp\""
Expand All @@ -64,15 +59,10 @@ for test in "${CHECK_PACKAGES_TESTS[@]}"; do
if [[ $test =~ with-kind$ ]]; then
echo " - build/kubectl-dump.txt"
fi
if [[ "${independent_agent}" == "false" ]]; then
echo " env:"
echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
fi
done
done

pushd test/packages/false_positives > /dev/null
for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
while IFS= read -r -d '' package ; do
package_name=$(basename "${package}")
echo " - label: \":go: Integration test (false positive): ${package_name}\""
echo " key: \"integration-false_positives-${package_name}\""
Expand All @@ -86,48 +76,56 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
echo " - build/test-results/*.xml"
echo " - build/test-results/*.xml.expected-errors.txt" # these files are uploaded in case it is needed to review the xUnit files in case of CI reports success the step
echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
done
done < <(find . -maxdepth 1 -mindepth 1 -type d -print0)
popd > /dev/null

pushd test/packages/parallel > /dev/null
for independent_agent in false true; do
for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
label_suffix=""
if [[ "$independent_agent" == "false" ]]; then
label_suffix=" (stack agent)"
fi
while IFS= read -r -d '' package ; do
package_name=$(basename "${package}")
if [[ "$independent_agent" == "false" && "$package_name" == "oracle" ]]; then
echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required software)."
continue
fi

if [[ "$independent_agent" == "false" && "$package_name" == "auditd_manager" ]]; then
echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing capabilities)."
continue
fi
echo " - label: \":go: Integration test: ${package_name}\""
echo " key: \"integration-parallel-${package_name}-agent\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
echo " env:"
echo " UPLOAD_SAFE_LOGS: 1"
echo " agents:"
echo " provider: \"gcp\""
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
echo " artifact_paths:"
echo " - build/test-results/*.xml"
echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
done < <(find . -maxdepth 1 -mindepth 1 -type d -print0)

if [[ "$independent_agent" == "false" && "$package_name" == "custom_entrypoint" ]]; then
echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required files deployed in provisioning)."
continue
fi
# Run system tests with the Elastic Agent from the Elastic stack just for one package
package_name="apache"
echo " - label: \":go: Integration test: ${package_name} (stack agent)\""
echo " key: \"integration-parallel-${package_name}-stack-agent\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
echo " env:"
echo " UPLOAD_SAFE_LOGS: 1"
echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: false"
echo " agents:"
echo " provider: \"gcp\""
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
echo " artifact_paths:"
echo " - build/test-results/*.xml"
echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package

echo " - label: \":go: Integration test: ${package_name}${label_suffix}\""
echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\""
# Add steps to test validation method mappings
while IFS= read -r -d '' package ; do
package_name=$(basename "${package}")
echo " - label: \":go: Integration test: ${package_name} (just validate mappings)\""
echo " key: \"integration-parallel-${package_name}-agent-validate-mappings\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
echo " env:"
echo " UPLOAD_SAFE_LOGS: 1"
if [[ "${independent_agent}" == "false" ]]; then
echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
fi
echo " ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: mappings"
echo " agents:"
echo " provider: \"gcp\""
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
echo " artifact_paths:"
echo " - build/test-results/*.xml"
echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
done
done
done < <(find . -maxdepth 1 -mindepth 1 -type d -print0)
popd > /dev/null

# TODO: Missing docker & docker-compose in MACOS ARM agent image, skip installation of packages in the meantime.
Expand Down Expand Up @@ -166,19 +164,11 @@ echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
echo " artifact_paths:"
echo " - build/elastic-stack-dump/install-zip-shellinit/logs/*.log"

for independent_agent in false true; do
label_suffix=""
if [[ "$independent_agent" == "false" ]]; then
label_suffix=" (stack agent)"
fi
echo " - label: \":go: Integration test: system-flags${label_suffix}\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags"
echo " agents:"
echo " provider: \"gcp\""
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
echo " env:"
echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
done
echo " - label: \":go: Integration test: system-flags\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags"
echo " agents:"
echo " provider: \"gcp\""
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""

echo " - label: \":go: Integration test: profiles-command\""
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"
Expand Down
4 changes: 4 additions & 0 deletions .buildkite/scripts/integration_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,10 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
package_folder="${package_folder}-stack_agent"
fi

if [[ "${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD:-""}" != "" ]]; then
package_folder="${package_folder}-${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD}"
fi

if [[ "${retry_count}" -ne 0 ]]; then
package_folder="${package_folder}_retry_${retry_count}"
fi
Expand Down
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -677,9 +677,16 @@ There are available some environment variables that could be used to change some
- `ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI`: If set to `true`, the Elastic Agent image used for running agents will be using the Ubuntu docker images
(e.g. `docker.elastic.co/elastic-agent/elastic-agent-complete`). If set to `false`, the Elastic Agent image used for the running agents will be based on the wolfi
images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `false`.
- `ELASTIC_PACKAGE_TEST_DUMP_SCENARIO_DOCS. If the variable is set, elastic-package will dump to a file the documents generated
- `ELASTIC_PACKAGE_TEST_DUMP_SCENARIO_DOCS`. If the variable is set, elastic-package will dump to a file the documents generated
by system tests before they are verified. This is useful to know exactly what fields are being verified when investigating
issues on this step. Documents are dumped to a file in the system temporary directory. It is disabled by default.
- `ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT`. If the variable is set to false, all system tests defined in the package will use
the Elastic Agent started along with the stack. If set to true, a new Elastic Agent will be started and enrolled for each test defined in the
package (and unenrolled at the end of each test). Default: `true`.
- `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `all`, `mappings` or `fields`. If this
variable is set to `fields`, then validation of fields will be based on the documents ingested into Elasticsearch. If this is set to
`mappings`, then validation of fields will be based on the mappings generated when the documents are ingested into Elasticsearch. If
set to `all`, then validation will be based on both methods mentioned previously. Default option: `fields`.

- To configure the Elastic stack to be used by `elastic-package`:
- `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)
Expand Down
82 changes: 82 additions & 0 deletions internal/elasticsearch/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,3 +279,85 @@ func (client *Client) redHealthCause(ctx context.Context) (string, error) {
}
return strings.Join(causes, ", "), nil
}

func (c *Client) SimulateIndexTemplate(ctx context.Context, indexTemplateName string) (json.RawMessage, json.RawMessage, error) {
resp, err := c.Indices.SimulateTemplate(
c.Indices.SimulateTemplate.WithContext(ctx),
c.Indices.SimulateTemplate.WithName(indexTemplateName),
)
if err != nil {
return nil, nil, fmt.Errorf("failed to get field mapping for data stream %q: %w", indexTemplateName, err)
}
defer resp.Body.Close()
if resp.IsError() {
return nil, nil, fmt.Errorf("error getting mapping: %s", resp)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, fmt.Errorf("error reading mapping body: %w", err)
}

type mappingsIndexTemplate struct {
DynamicTemplates json.RawMessage `json:"dynamic_templates"`
Properties json.RawMessage `json:"properties"`
}

type indexTemplateSimulated struct {
// Settings json.RawMessage `json:"settings"`
Mappings mappingsIndexTemplate `json:"mappings"`
}

type previewTemplate struct {
Template indexTemplateSimulated `json:"template"`
}

var preview previewTemplate

if err := json.Unmarshal(body, &preview); err != nil {
return nil, nil, fmt.Errorf("error unmarshaling mappings: %w", err)
}

return preview.Template.Mappings.DynamicTemplates, preview.Template.Mappings.Properties, nil
}

func (c *Client) DataStreamMappings(ctx context.Context, dataStreamName string) (json.RawMessage, json.RawMessage, error) {
mappingResp, err := c.Indices.GetMapping(
c.Indices.GetMapping.WithContext(ctx),
c.Indices.GetMapping.WithIndex(dataStreamName),
)
if err != nil {
return nil, nil, fmt.Errorf("failed to get field mapping for data stream %q: %w", dataStreamName, err)
}
defer mappingResp.Body.Close()
if mappingResp.IsError() {
return nil, nil, fmt.Errorf("error getting mapping: %s", mappingResp)
}
body, err := io.ReadAll(mappingResp.Body)
if err != nil {
return nil, nil, fmt.Errorf("error reading mapping body: %w", err)
}

type mappings struct {
DynamicTemplates json.RawMessage `json:"dynamic_templates"`
Properties json.RawMessage `json:"properties"`
}

mappingsRaw := map[string]struct {
Mappings mappings `json:"mappings"`
}{}

if err := json.Unmarshal(body, &mappingsRaw); err != nil {
return nil, nil, fmt.Errorf("error unmarshaling mappings: %w", err)
}

if len(mappingsRaw) != 1 {
return nil, nil, fmt.Errorf("exactly 1 mapping was expected, got %d", len(mappingsRaw))
}

var mappingsDefinition mappings
for _, v := range mappingsRaw {
mappingsDefinition = v.Mappings
}

return mappingsDefinition.DynamicTemplates, mappingsDefinition.Properties, nil
}
22 changes: 22 additions & 0 deletions internal/fields/dependency_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,31 @@ func parseECSFieldsSchema(content []byte) ([]FieldDefinition, error) {
return nil, fmt.Errorf("unmarshalling field body failed: %w", err)
}

// Force to set External as "ecs" in all these fields to be able to distinguish
// which fields come from ECS and which ones are loaded from the package directory
fields = setExternalAsECS(fields)

return fields, nil
}

func setExternalAsECS(fields []FieldDefinition) []FieldDefinition {
for i := 0; i < len(fields); i++ {
f := &fields[i]
f.External = "ecs"
if len(f.MultiFields) > 0 {
for j := 0; j < len(f.MultiFields); j++ {
mf := &f.MultiFields[j]
mf.External = "ecs"
}
}
if len(f.Fields) > 0 {
f.Fields = setExternalAsECS(f.Fields)
}
}

return fields
}

func asGitReference(reference string) (string, error) {
if !strings.HasPrefix(reference, gitReferencePrefix) {
return "", errors.New(`invalid Git reference ("git@" prefix expected)`)
Expand Down
52 changes: 52 additions & 0 deletions internal/fields/dependency_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -785,3 +785,55 @@ func TestDependencyManagerWithECS(t *testing.T) {
})
}
}

func TestValidate_SetExternalECS(t *testing.T) {
finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"}

validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first",
finder,
WithSpecVersion("2.3.0"),
WithEnabledImportAllECSSChema(true))
require.NoError(t, err)
require.NotNil(t, validator)

require.NotEmpty(t, validator.Schema)

cases := []struct {
title string
field string
external string
exists bool
}{
{
title: "field defined just in ECS",
field: "ecs.version",
external: "ecs",
exists: true,
},
{
title: "field defined fields directory package",
field: "service.status.duration.histogram",
external: "",
exists: true,
},
{
title: "undefined field",
field: "foo",
external: "",
exists: false,
},
}

for _, c := range cases {
t.Run(c.title, func(t *testing.T) {
found := FindElementDefinition(c.field, validator.Schema)
if !c.exists {
assert.Nil(t, found)
return
}

require.NotNil(t, found)
assert.Equal(t, c.external, found.External)
})
}
}
Loading

0 comments on commit 6def317

Please sign in to comment.