Skip to content

Commit

Permalink
Move installation package in policy and system tests (#1892)
Browse files Browse the repository at this point in the history 
This PR updates the runner to include two new methods in the interface
(SetupRunner and TearDownRunner). These two methods are in charge
of creating (or deleting) the global resources required to run all the tests
for every data streams and configuration file defined in the package.
Updated runners for system and policy tests.
  • Loading branch information
@mrodm
mrodm authored Jun 10, 2024
1 parent a5b607a commit 52cb28b
Show file tree
Hide file tree
Showing 8 changed files with 250 additions and 106 deletions.
87 changes: 52 additions & 35 deletions cmd/testrunner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error {

_, pkg := filepath.Split(packageRootPath)

runner := asset.NewAssetRunner(asset.AssetRunnerOptions{
runner := asset.NewAssetTester(asset.AssetTesterOptions{
TestFolder: testrunner.TestFolder{Package: pkg},
PackageRootPath: packageRootPath,
KibanaClient: kibanaClient,
Expand Down Expand Up @@ -289,17 +289,18 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error {
ctx, stop := signal.Enable(cmd.Context(), logger.Info)
defer stop()

var results []testrunner.TestResult
for _, folder := range testFolders {
runner := static.NewStaticRunner(static.StaticRunnerOptions{
factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
runner := static.NewStaticTester(static.StaticTesterOptions{
TestFolder: folder,
PackageRootPath: packageRootPath,
})
r, err := testrunner.Run(ctx, runner)
if err != nil {
return fmt.Errorf("error running package %s tests: %w", testType, err)
}
results = append(results, r...)

return runner, nil
}

results, err := testrunner.RunWithFactory(ctx, testFolders, factory)
if err != nil {
return err
}

return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
Expand Down Expand Up @@ -438,9 +439,8 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error {
return err
}

var results []testrunner.TestResult
for _, folder := range testFolders {
runner, err := pipeline.NewPipelineRunner(pipeline.PipelineRunnerOptions{
factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
runner, err := pipeline.NewPipelineTester(pipeline.PipelineTesterOptions{
Profile: profile,
TestFolder: folder,
PackageRootPath: packageRootPath,
Expand All @@ -451,14 +451,14 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error {
DeferCleanup: deferCleanup,
})
if err != nil {
return fmt.Errorf("failed to create pipeline runner: %w", err)
return nil, fmt.Errorf("failed to create pipeline runner: %w", err)
}
return runner, nil
}

r, err := testrunner.Run(ctx, runner)
if err != nil {
return fmt.Errorf("error running package %s tests: %w", testType, err)
}
results = append(results, r...)
results, err := testrunner.RunWithFactory(ctx, testFolders, factory)
if err != nil {
return err
}

return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
Expand Down Expand Up @@ -680,9 +680,16 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
cmd.Printf("Running tests per stages (technical preview)\n")
}

var results []testrunner.TestResult
for _, folder := range testFolders {
runner := system.NewSystemRunner(system.SystemRunnerOptions{
runner := system.NewSystemTestRunner(system.SystemTestRunnerOptions{
PackageRootPath: packageRootPath,
KibanaClient: kibanaClient,
RunSetup: runSetup,
RunTearDown: runTearDown,
RunTestsOnly: runTestsOnly,
})

factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
runner := system.NewSystemTester(system.SystemTesterOptions{
Profile: profile,
TestFolder: folder,
PackageRootPath: packageRootPath,
Expand All @@ -698,14 +705,19 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
RunIndependentElasticAgent: false,
})

r, err := testrunner.Run(ctx, runner)
if err != nil {
return fmt.Errorf("error running package %s tests: %w", testType, err)
}
results = append(results, r...)
return runner, nil
}

return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
results, err := testrunner.RunSuite(ctx, testFolders, runner, factory)
if err != nil {
return err
}

err = processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
if err != nil {
return fmt.Errorf("failed to process results: %w", err)
}
return nil
}

func getTestRunnerPolicyCommand() *cobra.Command {
Expand Down Expand Up @@ -832,19 +844,24 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
return fmt.Errorf("can't create Kibana client: %w", err)
}

var results []testrunner.TestResult
for _, folder := range testFolders {
runner := policy.NewPolicyRunner(policy.PolicyRunnerOptions{
runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{
PackageRootPath: packageRootPath,
KibanaClient: kibanaClient,
})

factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
runner := policy.NewPolicyTester(policy.PolicyTesterOptions{
TestFolder: folder,
PackageRootPath: packageRootPath,
GenerateTestResult: generateTestResult,
KibanaClient: kibanaClient,
})
r, err := testrunner.Run(ctx, runner)
if err != nil {
return fmt.Errorf("error running package %s tests: %w", testType, err)
}
results = append(results, r...)
return runner, nil
}

results, err := testrunner.RunSuite(ctx, testFolders, runner, factory)
if err != nil {
return err
}

return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
Expand Down
8 changes: 4 additions & 4 deletions internal/testrunner/runners/asset/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@ type runner struct {
resourcesManager *resources.Manager
}

type AssetRunnerOptions struct {
type AssetTesterOptions struct {
TestFolder testrunner.TestFolder
PackageRootPath string
KibanaClient *kibana.Client
}

func NewAssetRunner(options AssetRunnerOptions) *runner {
func NewAssetTester(options AssetTesterOptions) *runner {
runner := runner{
testFolder: options.TestFolder,
packageRootPath: options.PackageRootPath,
Expand All @@ -49,8 +49,8 @@ func NewAssetRunner(options AssetRunnerOptions) *runner {
return &runner
}

// Ensures that runner implements testrunner.TestRunner interface
var _ testrunner.TestRunner = new(runner)
// Ensures that runner implements testrunner.Tester interface
var _ testrunner.Tester = new(runner)

// Type returns the type of test that can be run by this test runner.
func (r *runner) Type() testrunner.TestType {
Expand Down
2 changes: 1 addition & 1 deletion internal/testrunner/runners/pipeline/coverage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
)

// getPipelineCoverage returns a coverage report for the provided set of ingest pipelines.
func getPipelineCoverage(options PipelineRunnerOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) {
func getPipelineCoverage(options PipelineTesterOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) {
dataStreamPath, found, err := packages.FindDataStreamRootForPath(options.TestFolder.Path)
if err != nil {
return nil, fmt.Errorf("locating data_stream root failed: %w", err)
Expand Down
10 changes: 5 additions & 5 deletions internal/testrunner/runners/pipeline/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ type runner struct {
provider stack.Provider
}

type PipelineRunnerOptions struct {
type PipelineTesterOptions struct {
Profile *profile.Profile
DeferCleanup time.Duration
API *elasticsearch.API
Expand All @@ -69,7 +69,7 @@ type PipelineRunnerOptions struct {
CoverageType string
}

func NewPipelineRunner(options PipelineRunnerOptions) (*runner, error) {
func NewPipelineTester(options PipelineTesterOptions) (*runner, error) {
r := runner{
profile: options.Profile,
deferCleanup: options.DeferCleanup,
Expand Down Expand Up @@ -110,8 +110,8 @@ type IngestPipelineReroute struct {
AdditionalFields map[string]interface{} `yaml:",inline"`
}

// Ensures that runner implements testrunner.TestRunner interface
var _ testrunner.TestRunner = new(runner)
// Ensures that runner implements testrunner.Tester interface
var _ testrunner.Tester = new(runner)

// Type returns the type of test that can be run by this test runner.
func (r *runner) Type() testrunner.TestType {
Expand Down Expand Up @@ -355,7 +355,7 @@ func (r *runner) runTestCase(ctx context.Context, testCaseFile string, dsPath st
}

if r.withCoverage {
tr.Coverage, err = getPipelineCoverage(PipelineRunnerOptions{
tr.Coverage, err = getPipelineCoverage(PipelineTesterOptions{
TestFolder: r.testFolder,
API: r.esAPI,
PackageRootPath: r.packageRootPath,
Expand Down
74 changes: 55 additions & 19 deletions internal/testrunner/runners/policy/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,72 @@ type runner struct {
packageRootPath string
generateTestResult bool
kibanaClient *kibana.Client

resourcesManager *resources.Manager
cleanup func(context.Context) error
}

type PolicyRunnerOptions struct {
// Ensures that runner implements testrunner.Tester interface
var _ testrunner.Tester = new(runner)

// Ensures that runner implements testrunner.TestRunner interface
var _ testrunner.TestRunner = new(runner)

type PolicyTestRunnerOptions struct {
KibanaClient *kibana.Client
PackageRootPath string
}

type PolicyTesterOptions struct {
TestFolder testrunner.TestFolder
KibanaClient *kibana.Client
PackageRootPath string
GenerateTestResult bool
}

func NewPolicyRunner(options PolicyRunnerOptions) *runner {
return &runner{
func NewPolicyTestRunner(options PolicyTestRunnerOptions) *runner {
runner := runner{
kibanaClient: options.KibanaClient,
packageRootPath: options.PackageRootPath,
}

runner.resourcesManager = resources.NewManager()
runner.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: runner.kibanaClient})
return &runner
}

func NewPolicyTester(options PolicyTesterOptions) *runner {
runner := runner{
kibanaClient: options.KibanaClient,
testFolder: options.TestFolder,
packageRootPath: options.PackageRootPath,
generateTestResult: options.GenerateTestResult,
}
runner.resourcesManager = resources.NewManager()
runner.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: runner.kibanaClient})
return &runner
}

// SetupRunner prepares global resources required by the test runner.
func (r *runner) SetupRunner(ctx context.Context) error {
cleanup, err := r.setupSuite(ctx, r.resourcesManager)
if err != nil {
return fmt.Errorf("failed to setup test runner: %w", err)
}
r.cleanup = cleanup

return nil
}

// TearDownRunner cleans up any global test runner resources. It must be called
// after the test runner has finished executing all its tests.
func (r *runner) TearDownRunner(ctx context.Context) error {
logger.Debug("Uninstalling package...")
err := r.cleanup(context.WithoutCancel(ctx))
if err != nil {
return fmt.Errorf("failed to clean up test runner: %w", err)
}
return nil
}

func (r *runner) Type() testrunner.TestType {
Expand All @@ -54,34 +104,19 @@ func (r *runner) String() string {
}

func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) {
manager := resources.NewManager()
manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{
Client: r.kibanaClient,
})

cleanup, err := r.setupSuite(ctx, manager)
if err != nil {
return nil, fmt.Errorf("failed to setup test runner: %w", err)
}

var results []testrunner.TestResult
tests, err := filepath.Glob(filepath.Join(r.testFolder.Path, "test-*.yml"))
if err != nil {
return nil, fmt.Errorf("failed to look for test files in %s: %w", r.testFolder.Path, err)
}
for _, test := range tests {
result, err := r.runTest(ctx, manager, test)
result, err := r.runTest(ctx, r.resourcesManager, test)
if err != nil {
logger.Error(err)
}
results = append(results, result...)
}

err = cleanup(context.WithoutCancel(ctx))
if err != nil {
return nil, fmt.Errorf("failed to clean up test runner: %w", err)
}

return results, nil
}

Expand Down Expand Up @@ -158,6 +193,7 @@ func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cl
return err
}

logger.Debugf("Installing package...")
_, err = manager.ApplyCtx(ctx, setupResources)
if err != nil {
if ctx.Err() == nil {
Expand Down
8 changes: 4 additions & 4 deletions internal/testrunner/runners/static/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,21 @@ type runner struct {
packageRootPath string
}

type StaticRunnerOptions struct {
type StaticTesterOptions struct {
TestFolder testrunner.TestFolder
PackageRootPath string
}

func NewStaticRunner(options StaticRunnerOptions) *runner {
func NewStaticTester(options StaticTesterOptions) *runner {
runner := runner{
testFolder: options.TestFolder,
packageRootPath: options.PackageRootPath,
}
return &runner
}

// Ensures that runner implements testrunner.TestRunner interface
var _ testrunner.TestRunner = new(runner)
// Ensures that runner implements testrunner.Tester interface
var _ testrunner.Tester = new(runner)

const (
// TestType defining asset loading tests
Expand Down
Loading

0 comments on commit 52cb28b

Please sign in to comment.