This repository has been archived by the owner on Sep 17, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: support for running feature files (#1193)
* chore: update unit tests docs in PR template * docs: document configurations location * docs: document how the CI spreads feature files and tags * chore: extract build logic to commons * chore: create build scripts per test suite This way each test suite will control its own life cycle * chore: update scripts used by CI * chore: remove extracted code from the original makefile * chore: remove unused variables in makefile * docs: update how to use the make goals * fix: move build-docs back to the original makefile * chore: move lint back to the original makefile * feat: support passing feature file to the test runner It will accept the file name of a feature file, otherwise, it will run all feature files under the 'features/' directory, as it did before this change. * docs: add features to the docs * docs: document TAGS and FEATURES as variables affecting the build * fix: update tag in docs * docs: include default value for features * chore: move SKIP_SCENARIOS next to related * docs: explain that SKIP is added to the TAGS variable * fix: typo * chore: grammar in docs Co-authored-by: cachedout <[email protected]> Co-authored-by: cachedout <[email protected]> Co-authored-by: Adam Stokes <[email protected]> (cherry picked from commit 888ecfe)
- Loading branch information
1 parent
47c27c4
commit 8e8b0ab
Showing
9 changed files
with
106 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,53 +1,8 @@ | ||
| include ../commons.mk | ||
| include ./commons-test.mk | ||
|
|
||
| SUITE?=metricbeat | ||
| TAGS?= | ||
| DEVELOPER_MODE?=false | ||
| ELASTIC_APM_ACTIVE?=false | ||
| FORMAT?=pretty | ||
| LOG_INCLUDE_TIMESTAMP?=TRUE | ||
| LOG_LEVEL?=INFO | ||
| TIMEOUT_FACTOR?=3 | ||
| #true by default, allowing developers to set SKIP_SCENARIOS=false | ||
| SKIP_SCENARIOS?=true | ||
| STACK_VERSION?= | ||
| PICKLES_VERSION?="2.20.1" | ||
| VERSION_VALUE=`cat ../cli/VERSION.txt` | ||
| ELASTIC_APM_ENVIRONMENT?=local | ||
|
|
||
| ifeq ($(ELASTIC_APM_ACTIVE),true) | ||
| ifeq ($(ELASTIC_APM_ENVIRONMENT),ci) | ||
| JENKINS_STATS_SECRET?=secret/observability-team/ci/jenkins-stats | ||
| export APM_SECRET_TOKEN?=$(shell vault read -field apmServerToken "$(JENKINS_STATS_SECRET)") | ||
| export APM_SERVER_URL?=$(shell vault read -field apmServerUrl "$(JENKINS_STATS_SECRET)") | ||
| endif | ||
|
|
||
| ifeq ($(ELASTIC_APM_ENVIRONMENT),local) | ||
| export APM_SECRET_TOKEN?= | ||
| export APM_SERVER_URL?=http://localhost:8200 | ||
| export ELASTIC_APM_GLOBAL_LABELS?= | ||
| endif | ||
| endif | ||
|
|
||
| ifneq ($(TAGS),) | ||
| TAGS_FLAG=--tags | ||
| ifeq ($(SKIP_SCENARIOS),true) | ||
| ## We always want to skip scenarios tagged with @skip | ||
| TAGS+= && ~skip | ||
| endif | ||
|
|
||
| # Double quote only if the tags are set | ||
| TAGS_VALUE="$(TAGS)" | ||
| else | ||
| ifeq ($(SKIP_SCENARIOS),true) | ||
| TAGS_FLAG=--tags | ||
| TAGS_VALUE="~skip" | ||
| endif | ||
| endif | ||
|
|
||
| GO_IMAGE_TAG?='stretch' | ||
| GOOS?='linux' | ||
| GOARCH?='amd64' | ||
|
|
||
| .PHONT: build-docs | ||
| build-docs: | ||
|
|
@@ -57,32 +12,6 @@ build-docs: | |
| mv docs/.\\/index.html docs/index.html | ||
| rm -fr docs/.\\ | ||
|
|
||
| .PHONY: install | ||
| install: | ||
| go get -v -t ./... | ||
|
|
||
| .PHONY: install-godog | ||
| install-godog: export GO111MODULE := on | ||
| install-godog: | ||
| go get -v github.com/cucumber/godog/cmd/[email protected] | ||
|
|
||
| .PHONY: functional-test | ||
| functional-test: install-godog | ||
| cd _suites/${SUITE} && \ | ||
| OP_LOG_LEVEL=${LOG_LEVEL} \ | ||
| OP_LOG_INCLUDE_TIMESTAMP=${LOG_INCLUDE_TIMESTAMP} \ | ||
| TIMEOUT_FACTOR=${TIMEOUT_FACTOR} \ | ||
| STACK_VERSION=${STACK_VERSION} \ | ||
| DEVELOPER_MODE=${DEVELOPER_MODE} \ | ||
| ELASTIC_APM_SERVICE_NAME="E2E Tests" \ | ||
| ELASTIC_APM_CENTRAL_CONFIG="false" \ | ||
| ELASTIC_APM_GLOBAL_LABELS=${ELASTIC_APM_GLOBAL_LABELS} \ | ||
| ELASTIC_APM_SERVICE_VERSION="${STACK_VERSION}" \ | ||
| ELASTIC_APM_ENVIRONMENT="${ELASTIC_APM_ENVIRONMENT}" \ | ||
| ELASTIC_APM_SECRET_TOKEN="${APM_SECRET_TOKEN}" \ | ||
| ELASTIC_APM_SERVER_URL="${APM_SERVER_URL}" \ | ||
| godog --format=${FORMAT} ${TAGS_FLAG} ${TAGS_VALUE} | ||
|
|
||
| .PHONY: lint | ||
| lint: | ||
| @docker run -t --rm -v $(PWD):/src -w /src gherkin/lint **/*.feature --disable AvoidOutlineForSingleExample,TooClumsy,TooManySteps,TooManyDifferentTags,TooLongStep,UseBackground | ||
|
|
@@ -91,16 +20,16 @@ lint: | |
|
|
||
| .PHONY: fleet-fleet | ||
| fleet-fleet: | ||
| SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test | ||
| TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) -C _suites/fleet functional-test | ||
|
|
||
| .PHONY: fleet-fleet-ci-snapshots | ||
| fleet-fleet-ci-snapshots: | ||
| SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test | ||
| TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) -C _suites/fleet functional-test | ||
|
|
||
| .PHONY: fleet-nightly | ||
| fleet-nightly: | ||
| SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test | ||
| TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) -C _suites/fleet functional-test | ||
|
|
||
| .PHONY: fleet-nightly-ci-snapshots | ||
| fleet-nightly-ci-snapshots: | ||
| SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test | ||
| TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) -C _suites/fleet functional-test | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| include ../../commons-test.mk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| include ../../commons-test.mk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| include ../../commons-test.mk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| include ../../commons-test.mk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| FEATURES?= | ||
| TAGS?= | ||
| DEVELOPER_MODE?=false | ||
| ELASTIC_APM_ACTIVE?=false | ||
| FORMAT?=pretty | ||
| LOG_INCLUDE_TIMESTAMP?=TRUE | ||
| LOG_LEVEL?=INFO | ||
| TIMEOUT_FACTOR?=3 | ||
| #true by default, allowing developers to set SKIP_SCENARIOS=false | ||
| SKIP_SCENARIOS?=true | ||
| STACK_VERSION?= | ||
|
|
||
| ELASTIC_APM_ENVIRONMENT?=local | ||
|
|
||
| ifeq ($(ELASTIC_APM_ACTIVE),true) | ||
| ifeq ($(ELASTIC_APM_ENVIRONMENT),ci) | ||
| JENKINS_STATS_SECRET?=secret/observability-team/ci/jenkins-stats | ||
| export APM_SECRET_TOKEN?=$(shell vault read -field apmServerToken "$(JENKINS_STATS_SECRET)") | ||
| export APM_SERVER_URL?=$(shell vault read -field apmServerUrl "$(JENKINS_STATS_SECRET)") | ||
| endif | ||
|
|
||
| ifeq ($(ELASTIC_APM_ENVIRONMENT),local) | ||
| export APM_SECRET_TOKEN?= | ||
| export APM_SERVER_URL?=http://localhost:8200 | ||
| export ELASTIC_APM_GLOBAL_LABELS?= | ||
| endif | ||
| endif | ||
|
|
||
| ifneq ($(FEATURES),) | ||
| FEATURES_VALUE=features/$(FEATURES) | ||
| else | ||
| FEATURES_VALUE= | ||
| endif | ||
|
|
||
| ifneq ($(TAGS),) | ||
| TAGS_FLAG=--tags | ||
| ifeq ($(SKIP_SCENARIOS),true) | ||
| ## We always want to skip scenarios tagged with @skip | ||
| TAGS+= && ~skip | ||
| endif | ||
|
|
||
| # Double quote only if the tags are set | ||
| TAGS_VALUE="$(TAGS)" | ||
| else | ||
| ifeq ($(SKIP_SCENARIOS),true) | ||
| TAGS_FLAG=--tags | ||
| TAGS_VALUE="~skip" | ||
| endif | ||
| endif | ||
|
|
||
| .PHONY: install | ||
| install: | ||
| go get -v -t ./... | ||
|
|
||
| .PHONY: install-godog | ||
| install-godog: export GO111MODULE := on | ||
| install-godog: | ||
| go get -v github.com/cucumber/godog/cmd/[email protected] | ||
|
|
||
| .PHONY: functional-test | ||
| functional-test: install-godog | ||
| OP_LOG_LEVEL=${LOG_LEVEL} \ | ||
| OP_LOG_INCLUDE_TIMESTAMP=${LOG_INCLUDE_TIMESTAMP} \ | ||
| TIMEOUT_FACTOR=${TIMEOUT_FACTOR} \ | ||
| STACK_VERSION=${STACK_VERSION} \ | ||
| DEVELOPER_MODE=${DEVELOPER_MODE} \ | ||
| ELASTIC_APM_SERVICE_NAME="E2E Tests" \ | ||
| ELASTIC_APM_CENTRAL_CONFIG="false" \ | ||
| ELASTIC_APM_GLOBAL_LABELS=${ELASTIC_APM_GLOBAL_LABELS} \ | ||
| ELASTIC_APM_SERVICE_VERSION="${STACK_VERSION}" \ | ||
| ELASTIC_APM_ENVIRONMENT="${ELASTIC_APM_ENVIRONMENT}" \ | ||
| ELASTIC_APM_SECRET_TOKEN="${APM_SECRET_TOKEN}" \ | ||
| ELASTIC_APM_SERVER_URL="${APM_SERVER_URL}" \ | ||
| godog --format=${FORMAT} ${FEATURES_VALUE} ${TAGS_FLAG} ${TAGS_VALUE} |