ci: E2E Framework [Core types] [1/6] (#2526)

add types

Author: matmerr

test/e2e/Makefile

@@ -0,0 +1,8 @@
+.PHONY: generate
+generate:
+	@go generate ./...
+
+.PHONY: acndev
+acndev: 
+	mkdir -p ./bin
+	go build -o ./bin/acndev ./cmd/

test/e2e/README.md

@@ -0,0 +1,25 @@
+# ACN E2E
+
+## Objectives
+- Steps are reusable
+- Steps parameters are saved to the context of the job
+- Once written to the job context, the values are immutable
+- Cluster resources used in code should be able to be generated to yaml for easy manual repro
+- Avoid shell/ps calls wherever possible and use go libraries for typed parameters (avoid capturing error codes/stderr/stdout)
+
+---
+## Starter Example:
+
+When authoring tests, make sure to prefix the test name with `TestE2E` so that it is skipped by existing pipeline unit test framework.
+For reference, see the `test-all` recipe in the root [Makefile](../../Makefile).
+
+
+For sample test, please check out:
+[the Hubble E2E.](./scenarios/hubble/index_test.go)
+
+
+## acndev CLI
+
+The `acndev` CLI is a tool for manually interacting with E2E steps for quick access. 
+
+It is used to create and manage clusters, but **not** to author tests with, and should **not** be referenced in pipeline yaml. Please stick to using tests with `TestE2E` prefix for authoring tests.

test/e2e/framework/types/job.go

@@ -0,0 +1,159 @@
+package types
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+)
+
+var (
+	ErrEmptyDescription    = fmt.Errorf("job description is empty")
+	ErrNonNilError         = fmt.Errorf("expected error to be non-nil")
+	ErrNilError            = fmt.Errorf("expected error to be nil")
+	ErrMissingParameter    = fmt.Errorf("missing parameter")
+	ErrParameterAlreadySet = fmt.Errorf("parameter already set")
+)
+
+type Job struct {
+	Values      *JobValues
+	Description string
+	Steps       []*StepWrapper
+}
+
+type StepWrapper struct {
+	Step Step
+	Opts *StepOptions
+}
+
+func responseDivider(jobname string) {
+	totalWidth := 100
+	start := 20
+	i := 0
+	for ; i < start; i++ {
+		fmt.Print("#")
+	}
+	mid := fmt.Sprintf(" %s ", jobname)
+	fmt.Print(mid)
+	for ; i < totalWidth-(start+len(mid)); i++ {
+		fmt.Print("#")
+	}
+	fmt.Println()
+}
+
+func NewJob(description string) *Job {
+	return &Job{
+		Values: &JobValues{
+			kv: make(map[string]string),
+		},
+		Description: description,
+	}
+}
+
+func (j *Job) AddScenario(steps ...StepWrapper) {
+	for _, step := range steps {
+		j.AddStep(step.Step, step.Opts)
+	}
+}
+
+func (j *Job) AddStep(step Step, opts *StepOptions) {
+	j.Steps = append(j.Steps, &StepWrapper{
+		Step: step,
+		Opts: opts,
+	})
+}
+
+func (j *Job) Run() error {
+	if j.Description == "" {
+		return ErrEmptyDescription
+	}
+
+	err := j.Validate()
+	if err != nil {
+		return err // nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself
+	}
+
+	for _, wrapper := range j.Steps {
+		err := wrapper.Step.Prevalidate()
+		if err != nil {
+			return err //nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself
+		}
+	}
+
+	for _, wrapper := range j.Steps {
+		responseDivider(reflect.TypeOf(wrapper.Step).Elem().Name())
+		log.Printf("INFO: step options provided: %+v\n", wrapper.Opts)
+		err := wrapper.Step.Run()
+		if wrapper.Opts.ExpectError && err == nil {
+			return fmt.Errorf("expected error from step %s but got nil: %w", reflect.TypeOf(wrapper.Step).Elem().Name(), ErrNilError)
+		} else if !wrapper.Opts.ExpectError && err != nil {
+			return fmt.Errorf("did not expect error from step %s but got error: %w", reflect.TypeOf(wrapper.Step).Elem().Name(), err)
+		}
+	}
+
+	for _, wrapper := range j.Steps {
+		err := wrapper.Step.Postvalidate()
+		if err != nil {
+			return err //nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself
+		}
+	}
+	return nil
+}
+
+func (j *Job) Validate() error {
+	for _, wrapper := range j.Steps {
+		err := j.validateStep(wrapper)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (j *Job) validateStep(stepw *StepWrapper) error {
+	stepName := reflect.TypeOf(stepw.Step).Elem().Name()
+	val := reflect.ValueOf(stepw.Step).Elem()
+
+	// set default options if none are provided
+	if stepw.Opts == nil {
+		stepw.Opts = &DefaultOpts
+	}
+
+	for i, f := range reflect.VisibleFields(val.Type()) {
+
+		// skip saving unexported fields
+		if !f.IsExported() {
+			continue
+		}
+
+		k := reflect.Indirect(val.Field(i)).Kind()
+
+		if k == reflect.String {
+			parameter := val.Type().Field(i).Name
+			value := val.Field(i).Interface().(string)
+			storedValue := j.Values.Get(parameter)
+
+			if storedValue == "" {
+				if value != "" {
+					if stepw.Opts.SaveParametersToJob {
+						fmt.Printf("%s setting parameter %s in job context to %s\n", stepName, parameter, value)
+						j.Values.Set(parameter, value)
+					}
+					continue
+				}
+				return fmt.Errorf("missing parameter %s for step %s: %w", parameter, stepName, ErrMissingParameter)
+
+			}
+
+			if value != "" {
+				return fmt.Errorf("parameter %s for step %s is already set from previous step: %w", parameter, stepName, ErrParameterAlreadySet)
+			}
+
+			// don't use log format since this is technically preexecution and easier to read
+			fmt.Println(stepName, "using previously stored value for parameter", parameter, "set as", j.Values.Get(parameter))
+			val.Field(i).SetString(storedValue)
+		}
+	}
+
+	return nil
+}

test/e2e/framework/types/jobvalues.go

@@ -0,0 +1,33 @@
+package types
+
+import "sync"
+
+type JobValues struct {
+	RWLock sync.RWMutex
+	kv     map[string]string
+}
+
+func (j *JobValues) New() *JobValues {
+	return &JobValues{
+		kv: make(map[string]string),
+	}
+}
+
+func (j *JobValues) Contains(key string) bool {
+	j.RWLock.RLock()
+	defer j.RWLock.RUnlock()
+	_, ok := j.kv[key]
+	return ok
+}
+
+func (j *JobValues) Get(key string) string {
+	j.RWLock.RLock()
+	defer j.RWLock.RUnlock()
+	return j.kv[key]
+}
+
+func (j *JobValues) Set(key, value string) {
+	j.RWLock.Lock()
+	defer j.RWLock.Unlock()
+	j.kv[key] = value
+}

test/e2e/framework/types/runner.go

@@ -0,0 +1,26 @@
+package types
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type Runner struct {
+	t   *testing.T
+	Job *Job
+}
+
+func NewRunner(t *testing.T, job *Job) *Runner {
+	return &Runner{
+		t:   t,
+		Job: job,
+	}
+}
+
+func (r *Runner) Run() {
+	if r.t.Failed() {
+		return
+	}
+	require.NoError(r.t, r.Job.Run())
+}

test/e2e/framework/types/step.go

@@ -0,0 +1,22 @@
+package types
+
+var DefaultOpts = StepOptions{
+	ExpectError:         false,
+	SaveParametersToJob: true,
+}
+
+type Step interface {
+	Prevalidate() error
+	Run() error
+	Postvalidate() error
+}
+
+type StepOptions struct {
+	ExpectError bool
+
+	// Generally set this to false when you want to reuse
+	// a step, but you don't want to save the parameters
+	// ex: Sleep for 15 seconds, then Sleep for 10 seconds,
+	// you don't want to save the parameters
+	SaveParametersToJob bool
+}

test/e2e/framework/types/step_sleep.go

@@ -0,0 +1,24 @@
+package types
+
+import (
+	"log"
+	"time"
+)
+
+type Sleep struct {
+	Duration time.Duration
+}
+
+func (c *Sleep) Run() error {
+	log.Printf("sleeping for %s...\n", c.Duration)
+	time.Sleep(c.Duration)
+	return nil
+}
+
+func (c *Sleep) Prevalidate() error {
+	return nil
+}
+
+func (c *Sleep) Postvalidate() error {
+	return nil
+}