diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index a0e2edd791..a1ef1f886d 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -31,4 +31,4 @@ mkdir -p outputs REPORT="$(pwd)/outputs/TEST-${SUITE}" -SUITE=${SUITE} TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} make --no-print-directory -C e2e functional-test +TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b1aa86f90a..90a0e98008 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -27,7 +27,7 @@ List here all the items you have verified BEFORE sending this PR. Please DO NOT - [ ] I have made corresponding changes to the documentation - [ ] I have made corresponding change to the default configuration files - [ ] I have added tests that prove my fix is effective or that my feature works -- [ ] I have run the Unit tests for the CLI, and they are passing locally +- [ ] I have run the Unit tests (`make unit-test`), and they are passing locally - [ ] I have run the End-2-End tests for the suite I'm working on, and they are passing locally - [ ] I have noticed new Go dependencies (run `make notice` in the proper directory) diff --git a/e2e/Makefile b/e2e/Makefile index 24a96d0908..9de24b51f7 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,53 +1,8 @@ include ../commons.mk +include ./commons-test.mk -SUITE?=metricbeat -TAGS?= -DEVELOPER_MODE?=false -ELASTIC_APM_ACTIVE?=false -FORMAT?=pretty -LOG_INCLUDE_TIMESTAMP?=TRUE -LOG_LEVEL?=INFO -TIMEOUT_FACTOR?=3 -#true by default, allowing developers to set SKIP_SCENARIOS=false -SKIP_SCENARIOS?=true -STACK_VERSION?= PICKLES_VERSION?="2.20.1" VERSION_VALUE=`cat ../cli/VERSION.txt` -ELASTIC_APM_ENVIRONMENT?=local - -ifeq ($(ELASTIC_APM_ACTIVE),true) -ifeq ($(ELASTIC_APM_ENVIRONMENT),ci) -JENKINS_STATS_SECRET?=secret/observability-team/ci/jenkins-stats -export APM_SECRET_TOKEN?=$(shell vault read -field apmServerToken "$(JENKINS_STATS_SECRET)") -export APM_SERVER_URL?=$(shell vault read -field apmServerUrl "$(JENKINS_STATS_SECRET)") -endif - -ifeq ($(ELASTIC_APM_ENVIRONMENT),local) -export APM_SECRET_TOKEN?= -export APM_SERVER_URL?=http://localhost:8200 -export ELASTIC_APM_GLOBAL_LABELS?= -endif -endif - -ifneq ($(TAGS),) -TAGS_FLAG=--tags -ifeq ($(SKIP_SCENARIOS),true) -## We always want to skip scenarios tagged with @skip -TAGS+= && ~skip -endif - -# Double quote only if the tags are set -TAGS_VALUE="$(TAGS)" -else -ifeq ($(SKIP_SCENARIOS),true) -TAGS_FLAG=--tags -TAGS_VALUE="~skip" -endif -endif - -GO_IMAGE_TAG?='stretch' -GOOS?='linux' -GOARCH?='amd64' .PHONT: build-docs build-docs: @@ -57,32 +12,6 @@ build-docs: mv docs/.\\/index.html docs/index.html rm -fr docs/.\\ -.PHONY: install -install: - go get -v -t ./... - -.PHONY: install-godog -install-godog: export GO111MODULE := on -install-godog: - go get -v github.com/cucumber/godog/cmd/godog@v0.11.0 - -.PHONY: functional-test -functional-test: install-godog - cd _suites/${SUITE} && \ - OP_LOG_LEVEL=${LOG_LEVEL} \ - OP_LOG_INCLUDE_TIMESTAMP=${LOG_INCLUDE_TIMESTAMP} \ - TIMEOUT_FACTOR=${TIMEOUT_FACTOR} \ - STACK_VERSION=${STACK_VERSION} \ - DEVELOPER_MODE=${DEVELOPER_MODE} \ - ELASTIC_APM_SERVICE_NAME="E2E Tests" \ - ELASTIC_APM_CENTRAL_CONFIG="false" \ - ELASTIC_APM_GLOBAL_LABELS=${ELASTIC_APM_GLOBAL_LABELS} \ - ELASTIC_APM_SERVICE_VERSION="${STACK_VERSION}" \ - ELASTIC_APM_ENVIRONMENT="${ELASTIC_APM_ENVIRONMENT}" \ - ELASTIC_APM_SECRET_TOKEN="${APM_SECRET_TOKEN}" \ - ELASTIC_APM_SERVER_URL="${APM_SERVER_URL}" \ - godog --format=${FORMAT} ${TAGS_FLAG} ${TAGS_VALUE} - .PHONY: lint lint: @docker run -t --rm -v $(PWD):/src -w /src gherkin/lint **/*.feature --disable AvoidOutlineForSingleExample,TooClumsy,TooManySteps,TooManyDifferentTags,TooLongStep,UseBackground @@ -91,16 +20,16 @@ lint: .PHONY: fleet-fleet fleet-fleet: - SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test + TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) -C _suites/fleet functional-test .PHONY: fleet-fleet-ci-snapshots fleet-fleet-ci-snapshots: - SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test + TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) -C _suites/fleet functional-test .PHONY: fleet-nightly fleet-nightly: - SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test + TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) -C _suites/fleet functional-test .PHONY: fleet-nightly-ci-snapshots fleet-nightly-ci-snapshots: - SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test + TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) -C _suites/fleet functional-test diff --git a/e2e/README.md b/e2e/README.md index ef4c7b196f..420f9a780a 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -82,8 +82,13 @@ The anatomy of a feature file is: - **But**: Used within any of the above clauses, it must tell an ocational reader a secondary preparation (Given), trigger (When), or output (Then) that must not be present. - **Examples:**: this `markdown table` will represent the elements to interpolate in the existing dynamic variables in the use case, being each column header the name of the different variables in the table. Besides that, each row will result in a test execution. +#### Feature files and the CI +There is [a descriptor file for the CI](../.ci/.e2e-tests.yaml) in which we define the parallel branches that will be created in the execution of a job. This YAML file defines suites and tags. A suite represents each test suite directory under the `e2e/_suites` directory, and the tags represent the tags will be passed to the test runner to filter the test execution. Another configuration we define in this file is related to the capabilities to run certain tags at the pull request stage, using the `pullRequestFilter` child element. This element will be appended to the tags used to filter the test runner. + +Adding a new feature file will require to check [the aforementioned descriptor file](../.ci/.e2e-tests.yaml). If the tags in the new file are not there, you should add a new parallel branch under the main test suite, or update the tags to add the new scenarios in an existing parallel branch. + ### Configuration files -It's possible that there will exist configuration YAML files in the test suire. We recommend locating them under the `configurations` folder in the suite directory. The name of the file will represent the feature to be tested (i.e. `apache.yml`). In this file we will add those configurations that are exclusive to the feature to be tests. +It's possible that the configuration YAML files will exist in the test suite. We recommend locating them under the `configurations` folder under the config directory for compose files (both profiles and services). The name of the file will represent the feature to be tested (i.e. `kibana.config.yml`). In this file we will add those configurations that are exclusive to the feature to be tested. ## Generating documentation about the specifications If you want to generate a website for the feature files, please run this command: @@ -149,7 +154,6 @@ The following environment variables affect how the tests are run in both the CI - `ELASTIC_APM_ENVIRONMENT`: Set this environment variable to `ci` to send APM data to Elastic Cloud. Otherwise, the framework will spin up local APM Server and Kibana instances. For the CI, it will read credentials from Vault. Default value: `local`. - `SKIP_PULL`: Set this environment variable to prevent the test suite to pull Docker images for all components. Default: `false` -- `SKIP_SCENARIOS`: Set this environment variable to `false` if it's needed to include the scenarios annotated as `@skip` in the current test execution. Default value: `true`. - `BEATS_LOCAL_PATH`: Set this environment variable to the base path to your local clone of Beats if it's needed to use the binary snapshots produced by your local build instead of the official releases. The snapshots will be fetched from the `${BEATS_LOCAL_PATH}/${THE_BEAT}/build/distributions` local directory. This variable is intended to be used by Beats developers, when testing locally the artifacts generated its own build. Default: empty. - `BEATS_USE_CI_SNAPSHOTS`: Set this environment variable to `true` if it's needed to use the binary snapshots produced by Beats CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. This variable is used by the Beats repository, when testing the artifacts generated by the packaging job. Default: `false`. - `LOG_LEVEL`: Set this environment variable to `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR` or `FATAL` to set the log level in the project. Default: `INFO`. @@ -159,6 +163,10 @@ The following environment variables affect how the tests are run in both the CI - **master (Fleet):** https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/e2e/_suites/fleet/ingest-manager_test.go#L39 - **master (Integrations):** https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/e2e/_suites/metricbeat/metricbeat_test.go#L30 - `TIMEOUT_FACTOR`: Set this environment variable to an integer number, which represents the factor to be used while waiting for resources within the tests. I.e. waiting for Kibana needs around 30 seconds. Instead of hardcoding 30 seconds, or 3 minutes, in the code, we use a backoff strategy to wait until an amount of time, specific per situation, multiplying it by the timeout factor. With that in mind, we are able to set a higher factor on CI without changing the code, and the developer is able to locally set specific conditions when running the tests on slower machines. Default: `3`. + +- `FEATURES`: Set this environment variable to an existing feature file, or a glob expression (`fleet_*.feature`), that will be passed to the test runner to filter the execution, selecting those feature files matching that expression. If empty, all feature files in the `features/` directory will be used. It can be used in combination with `TAGS`. +- `TAGS`: Set this environment variable to [a Cucumber tag expression](https://github.com/cucumber/godog#tags), that will be passed to the test runner to filter the execution, selecting those scenarios matching that expresion, across any feature file. It can be used in combination with `FEATURES`. +- `SKIP_SCENARIOS`: Set this environment variable to `false` if it's needed to include the scenarios annotated as `@skip` in the current test execution, adding that taf to the `TAGS` variable. Default value: `true`. ### Running regressions locally This example will run the Fleet tests for the 8.0.0-SNAPSHOT stack with the released 7.10.1 version of the agent. @@ -166,11 +174,19 @@ This example will run the Fleet tests for the 8.0.0-SNAPSHOT stack with the rele # Use the proper branch git checkout master # Run the tests for a specific branch -SUITE="fleet" \ +TAGS="fleet_mode_agent" \ + TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE \ + BEAT_VERSION="7.10.1" \ + make -C e2e/_suites/fleet functional-test +``` +Or running by feature file: +```shell +# Use the proper branch +git checkout master +FEATURES="fleet_mode_agent.feature" \ TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE \ - TAGS="fleet_mode" \ BEAT_VERSION="7.10.1" \ - make -C e2e functional-test + make -C e2e/_suites/fleet functional-test ``` When running regression testing locally, please make sure you clean up tool's workspace among runs. diff --git a/e2e/_suites/fleet/Makefile b/e2e/_suites/fleet/Makefile new file mode 100644 index 0000000000..c56d173398 --- /dev/null +++ b/e2e/_suites/fleet/Makefile @@ -0,0 +1 @@ +include ../../commons-test.mk diff --git a/e2e/_suites/helm/Makefile b/e2e/_suites/helm/Makefile new file mode 100644 index 0000000000..c56d173398 --- /dev/null +++ b/e2e/_suites/helm/Makefile @@ -0,0 +1 @@ +include ../../commons-test.mk diff --git a/e2e/_suites/kubernetes-autodiscover/Makefile b/e2e/_suites/kubernetes-autodiscover/Makefile new file mode 100644 index 0000000000..c56d173398 --- /dev/null +++ b/e2e/_suites/kubernetes-autodiscover/Makefile @@ -0,0 +1 @@ +include ../../commons-test.mk diff --git a/e2e/_suites/metricbeat/Makefile b/e2e/_suites/metricbeat/Makefile new file mode 100644 index 0000000000..c56d173398 --- /dev/null +++ b/e2e/_suites/metricbeat/Makefile @@ -0,0 +1 @@ +include ../../commons-test.mk diff --git a/e2e/commons-test.mk b/e2e/commons-test.mk new file mode 100644 index 0000000000..8b04a81bac --- /dev/null +++ b/e2e/commons-test.mk @@ -0,0 +1,74 @@ +FEATURES?= +TAGS?= +DEVELOPER_MODE?=false +ELASTIC_APM_ACTIVE?=false +FORMAT?=pretty +LOG_INCLUDE_TIMESTAMP?=TRUE +LOG_LEVEL?=INFO +TIMEOUT_FACTOR?=3 +#true by default, allowing developers to set SKIP_SCENARIOS=false +SKIP_SCENARIOS?=true +STACK_VERSION?= + +ELASTIC_APM_ENVIRONMENT?=local + +ifeq ($(ELASTIC_APM_ACTIVE),true) +ifeq ($(ELASTIC_APM_ENVIRONMENT),ci) +JENKINS_STATS_SECRET?=secret/observability-team/ci/jenkins-stats +export APM_SECRET_TOKEN?=$(shell vault read -field apmServerToken "$(JENKINS_STATS_SECRET)") +export APM_SERVER_URL?=$(shell vault read -field apmServerUrl "$(JENKINS_STATS_SECRET)") +endif + +ifeq ($(ELASTIC_APM_ENVIRONMENT),local) +export APM_SECRET_TOKEN?= +export APM_SERVER_URL?=http://localhost:8200 +export ELASTIC_APM_GLOBAL_LABELS?= +endif +endif + +ifneq ($(FEATURES),) +FEATURES_VALUE=features/$(FEATURES) +else +FEATURES_VALUE= +endif + +ifneq ($(TAGS),) +TAGS_FLAG=--tags +ifeq ($(SKIP_SCENARIOS),true) +## We always want to skip scenarios tagged with @skip +TAGS+= && ~skip +endif + +# Double quote only if the tags are set +TAGS_VALUE="$(TAGS)" +else +ifeq ($(SKIP_SCENARIOS),true) +TAGS_FLAG=--tags +TAGS_VALUE="~skip" +endif +endif + +.PHONY: install +install: + go get -v -t ./... + +.PHONY: install-godog +install-godog: export GO111MODULE := on +install-godog: + go get -v github.com/cucumber/godog/cmd/godog@v0.11.0 + +.PHONY: functional-test +functional-test: install-godog + OP_LOG_LEVEL=${LOG_LEVEL} \ + OP_LOG_INCLUDE_TIMESTAMP=${LOG_INCLUDE_TIMESTAMP} \ + TIMEOUT_FACTOR=${TIMEOUT_FACTOR} \ + STACK_VERSION=${STACK_VERSION} \ + DEVELOPER_MODE=${DEVELOPER_MODE} \ + ELASTIC_APM_SERVICE_NAME="E2E Tests" \ + ELASTIC_APM_CENTRAL_CONFIG="false" \ + ELASTIC_APM_GLOBAL_LABELS=${ELASTIC_APM_GLOBAL_LABELS} \ + ELASTIC_APM_SERVICE_VERSION="${STACK_VERSION}" \ + ELASTIC_APM_ENVIRONMENT="${ELASTIC_APM_ENVIRONMENT}" \ + ELASTIC_APM_SECRET_TOKEN="${APM_SECRET_TOKEN}" \ + ELASTIC_APM_SERVER_URL="${APM_SERVER_URL}" \ + godog --format=${FORMAT} ${FEATURES_VALUE} ${TAGS_FLAG} ${TAGS_VALUE}