From 761be75fd29a1ed166b6be333ca0927960185beb Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 18 Jan 2024 22:26:24 +0000 Subject: [PATCH 01/11] add scenarios --- .pipelines/networkobservability/e2e.yaml | 29 +++++ test/e2e/Makefile | 8 ++ test/e2e/README.md | 25 ++++ test/e2e/scenarios/cns/scenarios.go | 3 + test/e2e/scenarios/hubble/scenarios.go | 79 ++++++++++++ test/e2e/scenarios/hubble/scenarios_test.go | 62 +++++++++ .../hubble/steps/validate-drop-metric.go | 120 ++++++++++++++++++ .../hubble/steps/validate-metrics.go | 120 ++++++++++++++++++ 8 files changed, 446 insertions(+) create mode 100644 .pipelines/networkobservability/e2e.yaml create mode 100644 test/e2e/Makefile create mode 100644 test/e2e/README.md create mode 100644 test/e2e/scenarios/cns/scenarios.go create mode 100644 test/e2e/scenarios/hubble/scenarios.go create mode 100644 test/e2e/scenarios/hubble/scenarios_test.go create mode 100644 test/e2e/scenarios/hubble/steps/validate-drop-metric.go create mode 100644 test/e2e/scenarios/hubble/steps/validate-metrics.go diff --git a/.pipelines/networkobservability/e2e.yaml b/.pipelines/networkobservability/e2e.yaml new file mode 100644 index 0000000000..67406a8c50 --- /dev/null +++ b/.pipelines/networkobservability/e2e.yaml @@ -0,0 +1,29 @@ +pr: none + +stages: + - stage: Run_E2E + displayName: Run E2E + jobs: + - job: Run_E2E + displayName: Run Network Observability E2E + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + + steps: + - task: AzureCLI@2 + displayName: "Verify E2E Manifests" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + go test -v ./test/e2e/manifests/*.go + + - task: AzureCLI@2 + displayName: "Run Network Observability E2E" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + go test -v ./test/e2e/scenarios/hubble/*.go -timeout 30m diff --git a/test/e2e/Makefile b/test/e2e/Makefile new file mode 100644 index 0000000000..ac0827e525 --- /dev/null +++ b/test/e2e/Makefile @@ -0,0 +1,8 @@ +.PHONY: generate +generate: + @go generate ./... + +.PHONY: acndev +acndev: + mkdir -p ./bin + go build -o ./bin/acndev ./cmd/ diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000000..529c83145c --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,25 @@ +# ACN E2E + +## Objectives +- Steps are reusable +- Steps parameters are saved to the context of the job +- Once written to the job context, the values are immutable +- Cluster resources used in code should be able to be generated to yaml for easy manual repro +- Avoid shell/ps calls wherever possible and use go libraries for typed parameters (avoid capturing error codes/stderr/stdout) + +--- +## Starter Example: + +When authoring tests, make sure to prefix the test name with `TestE2E` so that it is skipped by existing pipeline unit test framework. +For reference, see the `test-all` recipe in the root [Makefile](../../Makefile). + + +For sample test, please check out: +[the Hubble E2E.](./scenarios/hubble/index_test.go) + + +## acndev CLI + +The `acndev` CLI is a tool for manually interacting with E2E steps for quick access. + +It is used to create and manage clusters, but **not** to author tests with, and should **not** be referenced in pipeline yaml. Please stick to using tests with `TestE2E` prefix for authoring tests. diff --git a/test/e2e/scenarios/cns/scenarios.go b/test/e2e/scenarios/cns/scenarios.go new file mode 100644 index 0000000000..8e4e36f7ca --- /dev/null +++ b/test/e2e/scenarios/cns/scenarios.go @@ -0,0 +1,3 @@ +package cns + +// reserved diff --git a/test/e2e/scenarios/hubble/scenarios.go b/test/e2e/scenarios/hubble/scenarios.go new file mode 100644 index 0000000000..2b5ec8a23c --- /dev/null +++ b/test/e2e/scenarios/hubble/scenarios.go @@ -0,0 +1,79 @@ +package hubble + +import ( + "time" + + k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" + "github.com/Azure/azure-container-networking/test/e2e/scenarios/hubble/steps" +) + +const ( + // Hubble drop reasons + UnsupportedL3Protocol = "UNSUPPORTED_L3_PROTOCOL" + PolicyDenied = "POLICY_DENIED" + + // L4 protocols + TCP = "TCP" + UDP = "UDP" + + Delay = 30 * time.Second +) + +func ValidateDropMetric() []types.StepWrapper { + return []types.StepWrapper{ + { + Step: &k8s.CreateKapingerDeployment{ + KapingerNamespace: "kube-system", + KapingerReplicas: "1", + }, + }, + { + Step: &k8s.CreateDenyAllNetworkPolicy{ + NetworkPolicyNamespace: "kube-system", + DenyAllLabelSelector: "app=agnhost-a", + }, + }, + { + Step: &k8s.CreateAgnhostStatefulSet{ + AgnhostName: "agnhost-a", + AgnhostNamespace: "kube-system", + }, + }, + { + Step: &k8s.ExecInPod{ + PodName: "agnhost-a-0", + PodNamespace: "kube-system", + Command: "curl -s -m 5 bing.com", + }, + Opts: &types.StepOptions{ + ExpectError: true, + }, + }, + { // metrics take some time to show up, so sleep for a bit + Step: &types.Sleep{ + Duration: Delay, + }, + Opts: &types.StepOptions{ + SaveParametersToJob: false, + }, + }, + { + Step: &k8s.PortForward{ + Namespace: "kube-system", + LabelSelector: "k8s-app=cilium", + LocalPort: "9965", + RemotePort: "9965", + OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace + }, + }, + { + Step: &steps.ValidateHubbleDropMetric{ + PortForwardedHubblePort: "9965", + Source: "agnhost-a", + Reason: PolicyDenied, + Protocol: UDP, + }, + }, + } +} diff --git a/test/e2e/scenarios/hubble/scenarios_test.go b/test/e2e/scenarios/hubble/scenarios_test.go new file mode 100644 index 0000000000..ad68d3501d --- /dev/null +++ b/test/e2e/scenarios/hubble/scenarios_test.go @@ -0,0 +1,62 @@ +package hubble + +import ( + "os" + "os/user" + "strconv" + "testing" + "time" + + "github.com/Azure/azure-container-networking/test/e2e/framework/azure" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" +) + +const ( + // netObsRGtag is used to tag resources created by this test suite + netObsRGtag = "-e2e-netobs-" +) + +// Test against a BYO cluster with Cilium and Hubble enabled, +// create a pod with a deny all network policy and validate +// that the drop metrics are present in the prometheus endpoint +func TestE2EDropHubbleMetrics(t *testing.T) { + job := types.NewJob("Validate that drop metrics are present in the prometheus endpoint") + runner := types.NewRunner(t, job) + defer runner.Run() + + curuser, _ := user.Current() + + testName := curuser.Username + netObsRGtag + strconv.FormatInt(time.Now().Unix(), 10) + sub := os.Getenv("AZURE_SUBSCRIPTION_ID") + + job.AddStep(&azure.CreateResourceGroup{ + SubscriptionID: sub, + ResourceGroupName: testName, + Location: "westus2", + }, nil) + + job.AddStep(&azure.CreateVNet{ + VnetName: "testvnet", + VnetAddressSpace: "10.0.0.0/9", + }, nil) + + job.AddStep(&azure.CreateSubnet{ + SubnetName: "testsubnet", + SubnetAddressSpace: "10.0.0.0/12", + }, nil) + + job.AddStep(&azure.CreateBYOCiliumCluster{ + ClusterName: testName, + PodCidr: "10.128.0.0/9", + DNSServiceIP: "192.168.0.10", + ServiceCidr: "192.168.0.0/28", + }, nil) + + job.AddStep(&azure.GetAKSKubeConfig{ + KubeConfigFilePath: "./test.pem", + }, nil) + + job.AddScenario(ValidateDropMetric()...) + + job.AddStep(&azure.DeleteResourceGroup{}, nil) +} diff --git a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go b/test/e2e/scenarios/hubble/steps/validate-drop-metric.go new file mode 100644 index 0000000000..51d2143f91 --- /dev/null +++ b/test/e2e/scenarios/hubble/steps/validate-drop-metric.go @@ -0,0 +1,120 @@ +package steps + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "reflect" + + promclient "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +var ErrNoMetricFound = fmt.Errorf("no metric found") + +const ( + destinationKey = "destination" + sourceKey = "source" + protcolKey = "protocol" + reason = "reason" +) + +type ValidateHubbleDropMetric struct { + PortForwardedHubblePort string // presumably port-forwarded to a cilium pod + Source string + Protocol string + Reason string +} + +func (v *ValidateHubbleDropMetric) Run() error { + promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.PortForwardedHubblePort) + ctx := context.Background() + pctx, cancel := context.WithCancel(ctx) + defer cancel() + + validMetric := map[string]string{ + destinationKey: "", + sourceKey: v.Source, + protcolKey: v.Protocol, + reason: v.Reason, + } + + metrics := map[string]*promclient.MetricFamily{} + scrapeMetricsFn := func() error { + log.Printf("attempting scrape metrics on %s", promAddress) + var err error + metrics, err = getPrometheusDropMetrics(promAddress) + if err != nil { + return fmt.Errorf("failed to get prometheus metrics: %w", err) + } + + return nil + } + + err := defaultRetrier.Do(pctx, scrapeMetricsFn) + if err != nil { + return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeoutSeconds, err) + } + + if !verifyLabelsPresent(metrics, validMetric) { + return fmt.Errorf("failed to find metric matching %+v: %w", validMetric, ErrNoMetricFound) + } + + log.Printf("found metric matching %+v\n", validMetric) + return nil +} + +func verifyLabelsPresent(data map[string]*promclient.MetricFamily, validMetric map[string]string) bool { + for _, metric := range data { + if metric.GetName() == "hubble_drop_total" { + for _, metric := range metric.GetMetric() { + + // get all labels and values on the metric + metricLabels := map[string]string{} + for _, label := range metric.GetLabel() { + metricLabels[label.GetName()] = label.GetValue() + } + if reflect.DeepEqual(metricLabels, validMetric) { + return true + } + } + } + } + + return false +} + +func getPrometheusDropMetrics(url string) (map[string]*promclient.MetricFamily, error) { + client := http.Client{} + resp, err := client.Get(url) //nolint + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request failed with status: %v", resp.Status) //nolint:goerr113,gocritic + } + + metrics, err := parseReaderPrometheusMetrics(resp.Body) + if err != nil { + return nil, err + } + + return metrics, nil +} + +func parseReaderPrometheusMetrics(input io.Reader) (map[string]*promclient.MetricFamily, error) { + var parser expfmt.TextParser + return parser.TextToMetricFamilies(input) //nolint +} + +func (v *ValidateHubbleDropMetric) Prevalidate() error { + return nil +} + +func (v *ValidateHubbleDropMetric) Postvalidate() error { + return nil +} diff --git a/test/e2e/scenarios/hubble/steps/validate-metrics.go b/test/e2e/scenarios/hubble/steps/validate-metrics.go new file mode 100644 index 0000000000..6ea4d9db20 --- /dev/null +++ b/test/e2e/scenarios/hubble/steps/validate-metrics.go @@ -0,0 +1,120 @@ +package steps + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "strings" + "time" + + "github.com/Azure/azure-container-networking/test/internal/retry" +) + +const ( + defaultTimeoutSeconds = 300 + defaultRetryDelay = 5 * time.Second + defaultRetryAttempts = 60 +) + +var ( + requiredMetrics = []string{ + "hubble_flows_processed_total", + "hubble_tcp_flags_total", + } + + defaultRetrier = retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} +) + +type ValidateHubbleFlowMetric struct { + LocalPort string +} + +func (v *ValidateHubbleFlowMetric) Run() error { + promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.LocalPort) + log.Printf("require all metrics to be present: %+v\n", requiredMetrics) + ctx := context.Background() + var metrics map[string]struct{} + scrapeMetricsFn := func() error { + log.Printf("attempting scrape metrics on %s", promAddress) + + var err error + metrics, err = getPrometheusMetrics(promAddress) + if err != nil { + return fmt.Errorf("failed to get prometheus metrics: %w", err) + } + return nil + } + + portForwardCtx, cancel := context.WithTimeout(ctx, defaultTimeoutSeconds*time.Second) + defer cancel() + + if err := defaultRetrier.Do(portForwardCtx, scrapeMetricsFn); err != nil { + return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeoutSeconds, err) + } + + for _, reqMetric := range requiredMetrics { + if _, exists := metrics[reqMetric]; !exists { + return fmt.Errorf("scraping %s, did not find metric %s: ", promAddress, reqMetric) //nolint:goerr113,gocritic + } + log.Printf("found metric %s\n", reqMetric) + } + + log.Printf("all metrics validated: %+v\n", requiredMetrics) + return nil +} + +func (v *ValidateHubbleFlowMetric) Prevalidate() error { + return nil +} + +func (v *ValidateHubbleFlowMetric) Postvalidate() error { + return nil +} + +func getPrometheusMetrics(url string) (map[string]struct{}, error) { + client := http.Client{} + resp, err := client.Get(url) //nolint + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request failed with status: %v", resp.Status) //nolint:goerr113,gocritic + } + + metricsData, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading HTTP response body failed: %w", err) + } + + metrics := parseMetrics(string(metricsData)) + return metrics, nil +} + +func parseMetrics(metricsData string) map[string]struct{} { + // Create a map to store the strings before the first '{'. + metrics := make(map[string]struct{}) + + // sample metrics + // hubble_tcp_flags_total{destination="",family="IPv4",flag="RST",source="kube-system/metrics-server"} 980 + // hubble_tcp_flags_total{destination="",family="IPv4",flag="SYN",source="kube-system/ama-metrics"} 1777 + // we only want the metric name for the time being + // label order/parseing can happen later + lines := strings.Split(metricsData, "\n") + // Iterate through each line. + for _, line := range lines { + // Find the index of the first '{' character. + index := strings.Index(line, "{") + if index >= 0 { + // Extract the string before the first '{'. + str := strings.TrimSpace(line[:index]) + // Store the string in the map. + metrics[str] = struct{}{} + } + } + + return metrics +} From 49f8fc8f823c34f1b7797cc3ddaa590535f68046 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 25 Jan 2024 23:32:15 +0000 Subject: [PATCH 02/11] port-forward and stitch hubble scenario --- .../framework/azure/create-cilium-cluster.go | 1 + .../kubernetes/create-agnhost-statefulset.go | 5 -- .../{port-forward.txt => port-forward.go} | 57 ++++++++++--------- test/e2e/framework/types/job.go | 11 +++- test/e2e/scenarios/hubble/scenarios.go | 10 +++- test/e2e/scenarios/hubble/scenarios_test.go | 2 +- .../networkobservability/hubble_test.go | 2 +- test/integration/portforward.go | 21 ++++++- 8 files changed, 70 insertions(+), 39 deletions(-) rename test/e2e/framework/kubernetes/{port-forward.txt => port-forward.go} (82%) diff --git a/test/e2e/framework/azure/create-cilium-cluster.go b/test/e2e/framework/azure/create-cilium-cluster.go index 671a25945b..d3edff65c7 100644 --- a/test/e2e/framework/azure/create-cilium-cluster.go +++ b/test/e2e/framework/azure/create-cilium-cluster.go @@ -113,6 +113,7 @@ func (c *CreateBYOCiliumCluster) Run() error { } log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + log.Printf("az aks get-credentials --resource-group %s --name %s --subscription %s", c.ResourceGroupName, c.ClusterName, c.SubscriptionID) poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil) if err != nil { diff --git a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go index 6be316c144..73a87c950e 100644 --- a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go +++ b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go @@ -19,11 +19,6 @@ var ErrLabelMissingFromPod = fmt.Errorf("label missing from pod") const ( AgnhostHTTPPort = 80 AgnhostReplicas = 1 - - defaultTimeoutSeconds = 300 - defaultRetryDelay = 5 * time.Second - defaultRetryAttempts = 60 - defaultHTTPClientTimeout = 2 * time.Second ) type CreateAgnhostStatefulSet struct { diff --git a/test/e2e/framework/kubernetes/port-forward.txt b/test/e2e/framework/kubernetes/port-forward.go similarity index 82% rename from test/e2e/framework/kubernetes/port-forward.txt rename to test/e2e/framework/kubernetes/port-forward.go index 7dcabe496d..e4fa0fc91a 100644 --- a/test/e2e/framework/kubernetes/port-forward.txt +++ b/test/e2e/framework/kubernetes/port-forward.go @@ -39,8 +39,7 @@ type PortForward struct { OptionalLabelAffinity string // local properties - pf *k8s.PortForwarder - portForwardHandle k8s.PortForwardStreamHandle + pf *k8s.PortForwarder } func (p *PortForward) Run() error { @@ -61,11 +60,6 @@ func (p *PortForward) Run() error { return fmt.Errorf("could not create clientset: %w", err) } - p.pf, err = k8s.NewPortForwarder(config) - if err != nil { - return fmt.Errorf("could not create port forwarder: %w", err) - } - // if we have an optional label affinity, find a pod with that label, on the same node as a pod with the label selector targetPodName := "" if p.OptionalLabelAffinity != "" { @@ -79,35 +73,40 @@ func (p *PortForward) Run() error { portForwardFn := func() error { log.Printf("attempting port forward to a pod with label \"%s\", in namespace \"%s\"...\n", p.LabelSelector, p.Namespace) - var handle k8s.PortForwardStreamHandle - // if we have a pod name (likely from affinity above), use it, otherwise use label selector + opts := k8s.PortForwardingOpts{ + Namespace: p.Namespace, + PodName: targetPodName, + LocalPort: lport, + DestPort: rport, + } + if targetPodName != "" { - handle, err = p.pf.ForwardWithPodName(pctx, p.Namespace, targetPodName, lport, rport) - if err != nil { - return fmt.Errorf("could not start port forward: %w", err) - } - } else { - handle, err = p.pf.ForwardWithLabelSelector(pctx, p.Namespace, p.LabelSelector, lport, rport) - if err != nil { - return fmt.Errorf("could not start port forward: %w", err) - } + opts.PodName = targetPodName + } + + p.pf, err = k8s.NewPortForwarder(config, &logger{}, opts) + if err != nil { + return fmt.Errorf("could not create port forwarder: %w", err) + } + err = p.pf.Forward(pctx) + if err != nil { + return fmt.Errorf("could not start port forward: %w", err) } // verify port forward succeeded client := http.Client{ Timeout: defaultHTTPClientTimeout, } - resp, err := client.Get(handle.URL()) //nolint + resp, err := client.Get(p.pf.Address()) //nolint if err != nil { - log.Printf("port forward validation HTTP request to %s failed: %v\n", handle.URL(), err) - handle.Stop() - return fmt.Errorf("port forward validation HTTP request to %s failed: %w", handle.URL(), err) + log.Printf("port forward validation HTTP request to %s failed: %v\n", p.pf.Address(), err) + p.pf.Stop() + return fmt.Errorf("port forward validation HTTP request to %s failed: %w", p.pf.Address(), err) } defer resp.Body.Close() - log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", handle.URL(), resp.Status) - p.portForwardHandle = handle + log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", p.pf.Address(), resp.Status) return nil } @@ -115,7 +114,7 @@ func (p *PortForward) Run() error { if err = defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil { return fmt.Errorf("could not start port forward within %ds: %w", defaultTimeoutSeconds, err) } - log.Printf("successfully port forwarded to \"%s\"\n", p.portForwardHandle.URL()) + log.Printf("successfully port forwarded to \"%s\"\n", p.pf.Address()) return nil } @@ -158,6 +157,12 @@ func (p *PortForward) Prevalidate() error { } func (p *PortForward) Postvalidate() error { - p.portForwardHandle.Stop() + p.pf.Stop() return nil } + +type logger struct{} + +func (l *logger) Logf(format string, args ...interface{}) { + log.Printf(format, args...) +} diff --git a/test/e2e/framework/types/job.go b/test/e2e/framework/types/job.go index eef5bb0b78..78c57c72fe 100644 --- a/test/e2e/framework/types/job.go +++ b/test/e2e/framework/types/job.go @@ -14,17 +14,24 @@ var ( ErrParameterAlreadySet = fmt.Errorf("parameter already set") ) +// A Job is a logical grouping of steps, options and values type Job struct { Values *JobValues Description string Steps []*StepWrapper } +// A StepWrapper is a coupling of a step and it's options type StepWrapper struct { Step Step Opts *StepOptions } +// A Scenario is a logical grouping of steps +type Scenario struct { + Steps []*StepWrapper +} + func responseDivider(jobname string) { totalWidth := 100 start := 20 @@ -49,8 +56,8 @@ func NewJob(description string) *Job { } } -func (j *Job) AddScenario(steps ...StepWrapper) { - for _, step := range steps { +func (j *Job) AddScenario(scenario *Scenario) { + for _, step := range scenario.Steps { j.AddStep(step.Step, step.Opts) } } diff --git a/test/e2e/scenarios/hubble/scenarios.go b/test/e2e/scenarios/hubble/scenarios.go index 2b5ec8a23c..0696d18606 100644 --- a/test/e2e/scenarios/hubble/scenarios.go +++ b/test/e2e/scenarios/hubble/scenarios.go @@ -20,8 +20,8 @@ const ( Delay = 30 * time.Second ) -func ValidateDropMetric() []types.StepWrapper { - return []types.StepWrapper{ +func ValidateDropMetric() *types.Scenario { + Steps := []*types.StepWrapper{ { Step: &k8s.CreateKapingerDeployment{ KapingerNamespace: "kube-system", @@ -76,4 +76,10 @@ func ValidateDropMetric() []types.StepWrapper { }, }, } + + scenario := &types.Scenario{ + Steps: Steps, + } + + return scenario } diff --git a/test/e2e/scenarios/hubble/scenarios_test.go b/test/e2e/scenarios/hubble/scenarios_test.go index ad68d3501d..28f873ca63 100644 --- a/test/e2e/scenarios/hubble/scenarios_test.go +++ b/test/e2e/scenarios/hubble/scenarios_test.go @@ -56,7 +56,7 @@ func TestE2EDropHubbleMetrics(t *testing.T) { KubeConfigFilePath: "./test.pem", }, nil) - job.AddScenario(ValidateDropMetric()...) + job.AddScenario(ValidateDropMetric()) job.AddStep(&azure.DeleteResourceGroup{}, nil) } diff --git a/test/integration/networkobservability/hubble_test.go b/test/integration/networkobservability/hubble_test.go index a363b9a97c..cae832368a 100644 --- a/test/integration/networkobservability/hubble_test.go +++ b/test/integration/networkobservability/hubble_test.go @@ -236,7 +236,7 @@ func TestEndpoints(t *testing.T) { portForwardFn := func() error { t.Logf("attempting port forward to a pod with label %s, in namespace %s...", labelSelector, namespace) - if err = pf.Forward(portForwardCtx); err != nil { + if err = pf.ForwardRandomPodWithOptsLabelSelector(portForwardCtx); err != nil { return fmt.Errorf("could not start port forward: %w", err) } return nil diff --git a/test/integration/portforward.go b/test/integration/portforward.go index 5170b91dc7..90c44ee211 100644 --- a/test/integration/portforward.go +++ b/test/integration/portforward.go @@ -39,6 +39,7 @@ type PortForwarder struct { type PortForwardingOpts struct { Namespace string LabelSelector string + PodName string LocalPort int DestPort int } @@ -71,7 +72,25 @@ func NewPortForwarder(restConfig *rest.Config, logger logger, opts PortForwardin // An error is returned if a port forwarding session could not be started. If no error is returned, the // Address method can be used to communicate with the pod, and the Stop and KeepAlive methods can be used // to manage the lifetime of the port forwarding session. + func (p *PortForwarder) Forward(ctx context.Context) error { + var podName string + if p.opts.PodName != "" { + pods, err := p.clientset.CoreV1().Pods(p.opts.Namespace).List(ctx, metav1.ListOptions{LabelSelector: p.opts.LabelSelector, FieldSelector: "status.phase=Running"}) + if err != nil { + return fmt.Errorf("could not list pods in %q with label %q: %w", p.opts.Namespace, p.opts.LabelSelector, err) + } + + if len(pods.Items) < 1 { + return fmt.Errorf("no pods found in %q with label %q", p.opts.Namespace, p.opts.LabelSelector) //nolint:goerr113 //no specific handling expected + } + + randomIndex := rand.Intn(len(pods.Items)) //nolint:gosec //this is going to be revised in the future anyways, avoid random pods + podName = pods.Items[randomIndex].Name + } else { + podName = p.opts.PodName + } + pods, err := p.clientset.CoreV1().Pods(p.opts.Namespace).List(ctx, metav1.ListOptions{LabelSelector: p.opts.LabelSelector, FieldSelector: "status.phase=Running"}) if err != nil { return fmt.Errorf("could not list pods in %q with label %q: %w", p.opts.Namespace, p.opts.LabelSelector, err) @@ -81,8 +100,6 @@ func (p *PortForwarder) Forward(ctx context.Context) error { return fmt.Errorf("no pods found in %q with label %q", p.opts.Namespace, p.opts.LabelSelector) //nolint:goerr113 //no specific handling expected } - randomIndex := rand.Intn(len(pods.Items)) //nolint:gosec //this is going to be revised in the future anyways, avoid random pods - podName := pods.Items[randomIndex].Name portForwardURL := p.clientset.CoreV1().RESTClient().Post(). Resource("pods"). Namespace(p.opts.Namespace). From 6168de12d00af52a3ce7c55e689d685895a0e4b1 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 23 Jan 2024 01:06:53 +0000 Subject: [PATCH 03/11] run jobs in background --- .../types/examples/background_test.go | 102 +++++++++++++++ test/e2e/framework/types/job.go | 119 +++++++++++++----- test/e2e/framework/types/step.go | 12 +- test/e2e/framework/types/step_sleep.go | 4 + test/e2e/framework/types/step_stop.go | 32 +++++ 5 files changed, 238 insertions(+), 31 deletions(-) create mode 100644 test/e2e/framework/types/examples/background_test.go create mode 100644 test/e2e/framework/types/step_stop.go diff --git a/test/e2e/framework/types/examples/background_test.go b/test/e2e/framework/types/examples/background_test.go new file mode 100644 index 0000000000..3abc64a3ef --- /dev/null +++ b/test/e2e/framework/types/examples/background_test.go @@ -0,0 +1,102 @@ +package types + +import ( + "fmt" + "log" + "sync" + "testing" + "time" + + "github.com/Azure/azure-container-networking/test/e2e/framework/types" +) + +func TestFramework(t *testing.T) { + job := types.NewJob("Validate that drop metrics are present in the prometheus endpoint") + runner := types.NewRunner(t, job) + defer runner.Run() + + job.AddStep(&TestBackground{ + CounterName: "Example Counter", + }, &types.StepOptions{ + ExpectError: false, + RunInBackgroundWithID: "TestStep", + }) + + job.AddStep(&types.Sleep{ + Duration: 1 * time.Second, + }, nil) + + job.AddStep(&types.Stop{ + BackgroundID: "TestStep", + }, nil) +} + +type TestBackground struct { + CounterName string + c *counter +} + +func (t *TestBackground) Run() error { + t.c = newCounter() + err := t.c.Start() + if err != nil { + return fmt.Errorf("failed to start counter: %w", err) + } + log.Println("running counter: " + t.CounterName) + return nil +} + +func (t *TestBackground) Stop() error { + log.Println("stopping counter: " + t.CounterName) + err := t.c.Stop() + if err != nil { + return fmt.Errorf("failed to stop counter: %w", err) + } + log.Println("count:", t.c.count) + return nil +} + +func (t *TestBackground) Prevalidate() error { + return nil +} + +func (t *TestBackground) Postvalidate() error { + return nil +} + +type counter struct { + ticker *time.Ticker + count int + ch chan struct{} + wg sync.WaitGroup +} + +func newCounter() *counter { + return &counter{ + ch: make(chan struct{}), + } +} + +func (c *counter) Start() error { + c.ticker = time.NewTicker(1 * time.Millisecond) + c.wg.Add(1) + go func() { + for { + select { + case <-c.ticker.C: + c.count++ + case <-c.ch: + c.wg.Done() + return + } + } + }() + + return nil +} + +func (c *counter) Stop() error { + close(c.ch) + c.wg.Wait() + return nil +} diff --git a/test/e2e/framework/types/job.go b/test/e2e/framework/types/job.go index 78c57c72fe..68ba984043 100644 --- a/test/e2e/framework/types/job.go +++ b/test/e2e/framework/types/job.go @@ -12,13 +12,16 @@ var ( ErrNilError = fmt.Errorf("expected error to be nil") ErrMissingParameter = fmt.Errorf("missing parameter") ErrParameterAlreadySet = fmt.Errorf("parameter already set") + ErrOrphanSteps = fmt.Errorf("background steps with no corresponding stop") + ErrCannotStopStep = fmt.Errorf("cannot stop step") ) // A Job is a logical grouping of steps, options and values type Job struct { - Values *JobValues - Description string - Steps []*StepWrapper + Values *JobValues + Description string + Steps []*StepWrapper + BackgroundSteps map[string]*StepWrapper } // A StepWrapper is a coupling of a step and it's options @@ -52,7 +55,8 @@ func NewJob(description string) *Job { Values: &JobValues{ kv: make(map[string]string), }, - Description: description, + BackgroundSteps: make(map[string]*StepWrapper), + Description: description, } } @@ -63,10 +67,11 @@ func (j *Job) AddScenario(scenario *Scenario) { } func (j *Job) AddStep(step Step, opts *StepOptions) { - j.Steps = append(j.Steps, &StepWrapper{ + stepw := &StepWrapper{ Step: step, Opts: opts, - }) + } + j.Steps = append(j.Steps, stepw) } func (j *Job) Run() error { @@ -74,6 +79,7 @@ func (j *Job) Run() error { return ErrEmptyDescription } + // validate all steps in the job, making sure parameters are set/validated etc. err := j.Validate() if err != nil { return err // nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself @@ -107,11 +113,58 @@ func (j *Job) Run() error { } func (j *Job) Validate() error { + // ensure that there are no background steps left after running + for _, wrapper := range j.Steps { err := j.validateStep(wrapper) if err != nil { return err } + + } + + err := j.validateBackgroundSteps() + if err != nil { + return err + } + + return nil +} + +func (j *Job) validateBackgroundSteps() error { + stoppedBackgroundSteps := make(map[string]bool) + + for _, stepw := range j.Steps { + switch s := stepw.Step.(type) { + case *Stop: + if j.BackgroundSteps[s.BackgroundID] == nil { + return fmt.Errorf("cannot stop step %s, as it won't be started by this time; %w", s.BackgroundID, ErrCannotStopStep) + } + if stopped := stoppedBackgroundSteps[s.BackgroundID]; stopped { + return fmt.Errorf("cannot stop step %s, as it has already been stopped; %w", s.BackgroundID, ErrCannotStopStep) + } + + // track for later on if the stop step is called + stoppedBackgroundSteps[s.BackgroundID] = true + + // set the stop step within the step + s.Step = j.BackgroundSteps[s.BackgroundID].Step + + default: + if stepw.Opts.RunInBackgroundWithID != "" { + if _, exists := j.BackgroundSteps[stepw.Opts.RunInBackgroundWithID]; exists { + log.Fatalf("step with id %s already exists", stepw.Opts.RunInBackgroundWithID) + } + j.BackgroundSteps[stepw.Opts.RunInBackgroundWithID] = stepw + stoppedBackgroundSteps[stepw.Opts.RunInBackgroundWithID] = false + } + } + } + + for stepName, stopped := range stoppedBackgroundSteps { + if !stopped { + return fmt.Errorf("step %s was not stopped; %w", stepName, ErrOrphanSteps) + } } return nil @@ -126,41 +179,51 @@ func (j *Job) validateStep(stepw *StepWrapper) error { stepw.Opts = &DefaultOpts } - for i, f := range reflect.VisibleFields(val.Type()) { + switch stepw.Step.(type) { + case *Stop: + // don't validate stop steps + return nil - // skip saving unexported fields - if !f.IsExported() { - continue - } + case *Sleep: + // don't validate sleep steps + return nil + + default: + for i, f := range reflect.VisibleFields(val.Type()) { - k := reflect.Indirect(val.Field(i)).Kind() + // skip saving unexported fields + if !f.IsExported() { + continue + } - if k == reflect.String { - parameter := val.Type().Field(i).Name - value := val.Field(i).Interface().(string) - storedValue := j.Values.Get(parameter) + k := reflect.Indirect(val.Field(i)).Kind() + + if k == reflect.String { + parameter := val.Type().Field(i).Name + value := val.Field(i).Interface().(string) + storedValue := j.Values.Get(parameter) + + if storedValue == "" { + if value != "" { - if storedValue == "" { - if value != "" { - if stepw.Opts.SaveParametersToJob { fmt.Printf("%s setting parameter %s in job context to %s\n", stepName, parameter, value) j.Values.Set(parameter, value) + + } else { + return fmt.Errorf("missing parameter %s for step %s: %w", parameter, stepName, ErrMissingParameter) } continue } - return fmt.Errorf("missing parameter %s for step %s: %w", parameter, stepName, ErrMissingParameter) - } + if value != "" { + return fmt.Errorf("parameter %s for step %s is already set from previous step: %w", parameter, stepName, ErrParameterAlreadySet) + } - if value != "" { - return fmt.Errorf("parameter %s for step %s is already set from previous step: %w", parameter, stepName, ErrParameterAlreadySet) + // don't use log format since this is technically preexecution and easier to read + fmt.Println(stepName, "using previously stored value for parameter", parameter, "set as", j.Values.Get(parameter)) + val.Field(i).SetString(storedValue) } - - // don't use log format since this is technically preexecution and easier to read - fmt.Println(stepName, "using previously stored value for parameter", parameter, "set as", j.Values.Get(parameter)) - val.Field(i).SetString(storedValue) } } - return nil } diff --git a/test/e2e/framework/types/step.go b/test/e2e/framework/types/step.go index 096de78970..fc82cb1ae2 100644 --- a/test/e2e/framework/types/step.go +++ b/test/e2e/framework/types/step.go @@ -1,14 +1,15 @@ package types var DefaultOpts = StepOptions{ - ExpectError: false, - SaveParametersToJob: true, + ExpectError: false, + SkipSavingParamatersToJob: false, } type Step interface { Prevalidate() error Run() error Postvalidate() error + Stop() error } type StepOptions struct { @@ -18,5 +19,10 @@ type StepOptions struct { // a step, but you don't want to save the parameters // ex: Sleep for 15 seconds, then Sleep for 10 seconds, // you don't want to save the parameters - SaveParametersToJob bool + SkipSavingParamatersToJob bool + + // Will save this step to the job's steps + // and then later on when Stop is called with job name, + // it will call Stop() on the step + RunInBackgroundWithID string } diff --git a/test/e2e/framework/types/step_sleep.go b/test/e2e/framework/types/step_sleep.go index b65f7bfeaf..1751fe7367 100644 --- a/test/e2e/framework/types/step_sleep.go +++ b/test/e2e/framework/types/step_sleep.go @@ -15,6 +15,10 @@ func (c *Sleep) Run() error { return nil } +func (c *Sleep) Stop() error { + return nil +} + func (c *Sleep) Prevalidate() error { return nil } diff --git a/test/e2e/framework/types/step_stop.go b/test/e2e/framework/types/step_stop.go new file mode 100644 index 0000000000..9a7ee3f151 --- /dev/null +++ b/test/e2e/framework/types/step_stop.go @@ -0,0 +1,32 @@ +package types + +import ( + "fmt" + "reflect" +) + +type Stop struct { + BackgroundID string + Step Step +} + +func (c *Stop) Run() error { + err := c.Step.Stop() + if err != nil { + stepName := reflect.TypeOf(c.Step).Elem().Name() + return fmt.Errorf("failed to stop step: %s with err %w", stepName, err) + } + return nil +} + +func (c *Stop) Stop() error { + return nil +} + +func (c *Stop) Prevalidate() error { + return nil +} + +func (c *Stop) Postvalidate() error { + return nil +} From d195f72e8ab9bdf669dd4ae73836ac61012b8876 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 25 Jan 2024 23:57:02 +0000 Subject: [PATCH 04/11] import stop to steps --- test/e2e/framework/azure/create-cilium-cluster.go | 7 ++++++- test/e2e/framework/azure/create-cluster.go | 4 ++++ test/e2e/framework/azure/create-rg.go | 4 ++++ test/e2e/framework/azure/create-vnet.go | 8 ++++++++ test/e2e/framework/azure/delete-cluster.go | 4 ++++ test/e2e/framework/azure/delete-rg.go | 4 ++++ test/e2e/framework/azure/enable-ama.go | 4 ++++ test/e2e/framework/azure/get-kubeconfig.go | 4 ++++ .../kubernetes/create-agnhost-statefulset.go | 4 ++++ .../kubernetes/create-kapinger-deployment.go | 4 ++++ .../e2e/framework/kubernetes/create-network-policy.go | 4 ++++ test/e2e/framework/kubernetes/exec-pod.go | 4 ++++ test/e2e/framework/kubernetes/port-forward.go | 5 +++++ test/e2e/framework/types/job.go | 5 +++++ test/e2e/scenarios/hubble/scenarios.go | 11 ++++++++--- .../scenarios/hubble/steps/validate-drop-metric.go | 4 ++++ 16 files changed, 76 insertions(+), 4 deletions(-) diff --git a/test/e2e/framework/azure/create-cilium-cluster.go b/test/e2e/framework/azure/create-cilium-cluster.go index d3edff65c7..e4a8320a7b 100644 --- a/test/e2e/framework/azure/create-cilium-cluster.go +++ b/test/e2e/framework/azure/create-cilium-cluster.go @@ -72,6 +72,10 @@ func (c *CreateBYOCiliumCluster) Postvalidate() error { return nil } +func (c *CreateBYOCiliumCluster) Stop() error { + return nil +} + func (c *CreateBYOCiliumCluster) Run() error { // Start with default cluster template ciliumCluster := GetStarterClusterTemplate(c.Location) @@ -112,8 +116,9 @@ func (c *CreateBYOCiliumCluster) Run() error { return fmt.Errorf("failed to create az client: %w", err) } - log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + log.Printf("when the cluster is ready, use the below command to access and debug") log.Printf("az aks get-credentials --resource-group %s --name %s --subscription %s", c.ResourceGroupName, c.ClusterName, c.SubscriptionID) + log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil) if err != nil { diff --git a/test/e2e/framework/azure/create-cluster.go b/test/e2e/framework/azure/create-cluster.go index 54250bbabc..3226c3a44f 100644 --- a/test/e2e/framework/azure/create-cluster.go +++ b/test/e2e/framework/azure/create-cluster.go @@ -113,3 +113,7 @@ func (c *CreateCluster) Prevalidate() error { func (c *CreateCluster) Postvalidate() error { return nil } + +func (c *CreateCluster) Stop() error { + return nil +} diff --git a/test/e2e/framework/azure/create-rg.go b/test/e2e/framework/azure/create-rg.go index efa1d88ea5..d4de2775af 100644 --- a/test/e2e/framework/azure/create-rg.go +++ b/test/e2e/framework/azure/create-rg.go @@ -46,3 +46,7 @@ func (c *CreateResourceGroup) Prevalidate() error { func (c *CreateResourceGroup) Postvalidate() error { return nil } + +func (c *CreateResourceGroup) Stop() error { + return nil +} diff --git a/test/e2e/framework/azure/create-vnet.go b/test/e2e/framework/azure/create-vnet.go index 0149a14c8d..a1ade96e53 100644 --- a/test/e2e/framework/azure/create-vnet.go +++ b/test/e2e/framework/azure/create-vnet.go @@ -63,6 +63,10 @@ func (c *CreateVNet) Postvalidate() error { return nil } +func (c *CreateVNet) Stop() error { + return nil +} + type CreateSubnet struct { SubscriptionID string ResourceGroupName string @@ -108,3 +112,7 @@ func (c *CreateSubnet) Prevalidate() error { func (c *CreateSubnet) Postvalidate() error { return nil } + +func (c *CreateSubnet) Stop() error { + return nil +} diff --git a/test/e2e/framework/azure/delete-cluster.go b/test/e2e/framework/azure/delete-cluster.go index 674a2fbb62..a3febc60bd 100644 --- a/test/e2e/framework/azure/delete-cluster.go +++ b/test/e2e/framework/azure/delete-cluster.go @@ -46,3 +46,7 @@ func (d *DeleteCluster) Prevalidate() error { func (d *DeleteCluster) Postvalidate() error { return nil } + +func (d *DeleteCluster) Stop() error { + return nil +} diff --git a/test/e2e/framework/azure/delete-rg.go b/test/e2e/framework/azure/delete-rg.go index 0d65df6d5f..fbeb8b4da6 100644 --- a/test/e2e/framework/azure/delete-rg.go +++ b/test/e2e/framework/azure/delete-rg.go @@ -50,3 +50,7 @@ func (d *DeleteResourceGroup) Prevalidate() error { func (d *DeleteResourceGroup) Postvalidate() error { return nil } + +func (d *DeleteResourceGroup) Stop() error { + return nil +} diff --git a/test/e2e/framework/azure/enable-ama.go b/test/e2e/framework/azure/enable-ama.go index e6bdaf4e61..f8346c4039 100644 --- a/test/e2e/framework/azure/enable-ama.go +++ b/test/e2e/framework/azure/enable-ama.go @@ -115,3 +115,7 @@ func (c *CreateAzureMonitor) Prevalidate() error { func (c *CreateAzureMonitor) Postvalidate() error { return nil } + +func (c *CreateAzureMonitor) Stop() error { + return nil +} diff --git a/test/e2e/framework/azure/get-kubeconfig.go b/test/e2e/framework/azure/get-kubeconfig.go index 08cb841403..7651f47a5b 100644 --- a/test/e2e/framework/azure/get-kubeconfig.go +++ b/test/e2e/framework/azure/get-kubeconfig.go @@ -51,3 +51,7 @@ func (c *GetAKSKubeConfig) Prevalidate() error { func (c *GetAKSKubeConfig) Postvalidate() error { return nil } + +func (c *GetAKSKubeConfig) Stop() error { + return nil +} diff --git a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go index 73a87c950e..f86223630b 100644 --- a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go +++ b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go @@ -70,6 +70,10 @@ func (c *CreateAgnhostStatefulSet) Postvalidate() error { return nil } +func (c *CreateAgnhostStatefulSet) Stop() error { + return nil +} + func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet { reps := int32(AgnhostReplicas) diff --git a/test/e2e/framework/kubernetes/create-kapinger-deployment.go b/test/e2e/framework/kubernetes/create-kapinger-deployment.go index b65732d2ee..abe8f7a200 100644 --- a/test/e2e/framework/kubernetes/create-kapinger-deployment.go +++ b/test/e2e/framework/kubernetes/create-kapinger-deployment.go @@ -74,6 +74,10 @@ func (c *CreateKapingerDeployment) Postvalidate() error { return nil } +func (c *CreateKapingerDeployment) Stop() error { + return nil +} + func (c *CreateKapingerDeployment) GetKapingerDeployment() *appsv1.Deployment { replicas, err := strconv.ParseInt(c.KapingerReplicas, 10, 32) if err != nil { diff --git a/test/e2e/framework/kubernetes/create-network-policy.go b/test/e2e/framework/kubernetes/create-network-policy.go index fa1171a257..900cc45f47 100644 --- a/test/e2e/framework/kubernetes/create-network-policy.go +++ b/test/e2e/framework/kubernetes/create-network-policy.go @@ -76,6 +76,10 @@ func (c *CreateDenyAllNetworkPolicy) Postvalidate() error { return nil } +func (c *CreateDenyAllNetworkPolicy) Stop() error { + return nil +} + type DeleteDenyAllNetworkPolicy struct { NetworkPolicyNamespace string KubeConfigFilePath string diff --git a/test/e2e/framework/kubernetes/exec-pod.go b/test/e2e/framework/kubernetes/exec-pod.go index aaf71c4945..1ce916a7c0 100644 --- a/test/e2e/framework/kubernetes/exec-pod.go +++ b/test/e2e/framework/kubernetes/exec-pod.go @@ -43,6 +43,10 @@ func (e *ExecInPod) Postvalidate() error { return nil } +func (e *ExecInPod) Stop() error { + return nil +} + func ExecPod(ctx context.Context, kubeConfigFilePath, namespace, podName, command string) error { config, err := clientcmd.BuildConfigFromFlags("", kubeConfigFilePath) if err != nil { diff --git a/test/e2e/framework/kubernetes/port-forward.go b/test/e2e/framework/kubernetes/port-forward.go index e4fa0fc91a..423eb0f4a1 100644 --- a/test/e2e/framework/kubernetes/port-forward.go +++ b/test/e2e/framework/kubernetes/port-forward.go @@ -161,6 +161,11 @@ func (p *PortForward) Postvalidate() error { return nil } +func (p *PortForward) Stop() error { + p.pf.Stop() + return nil +} + type logger struct{} func (l *logger) Logf(format string, args ...interface{}) { diff --git a/test/e2e/framework/types/job.go b/test/e2e/framework/types/job.go index 68ba984043..0ed66606f4 100644 --- a/test/e2e/framework/types/job.go +++ b/test/e2e/framework/types/job.go @@ -14,6 +14,7 @@ var ( ErrParameterAlreadySet = fmt.Errorf("parameter already set") ErrOrphanSteps = fmt.Errorf("background steps with no corresponding stop") ErrCannotStopStep = fmt.Errorf("cannot stop step") + ErrMissingBackroundID = fmt.Errorf("missing background id") ) // A Job is a logical grouping of steps, options and values @@ -137,6 +138,10 @@ func (j *Job) validateBackgroundSteps() error { for _, stepw := range j.Steps { switch s := stepw.Step.(type) { case *Stop: + if s.BackgroundID == "" { + return fmt.Errorf("cannot stop step with empty background id; %w", ErrMissingBackroundID) + } + if j.BackgroundSteps[s.BackgroundID] == nil { return fmt.Errorf("cannot stop step %s, as it won't be started by this time; %w", s.BackgroundID, ErrCannotStopStep) } diff --git a/test/e2e/scenarios/hubble/scenarios.go b/test/e2e/scenarios/hubble/scenarios.go index 0696d18606..ff1379dc82 100644 --- a/test/e2e/scenarios/hubble/scenarios.go +++ b/test/e2e/scenarios/hubble/scenarios.go @@ -54,9 +54,6 @@ func ValidateDropMetric() *types.Scenario { Step: &types.Sleep{ Duration: Delay, }, - Opts: &types.StepOptions{ - SaveParametersToJob: false, - }, }, { Step: &k8s.PortForward{ @@ -66,6 +63,9 @@ func ValidateDropMetric() *types.Scenario { RemotePort: "9965", OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace }, + Opts: &types.StepOptions{ + RunInBackgroundWithID: "hubble-drop-port-forward", + }, }, { Step: &steps.ValidateHubbleDropMetric{ @@ -75,6 +75,11 @@ func ValidateDropMetric() *types.Scenario { Protocol: UDP, }, }, + { + Step: &types.Stop{ + BackgroundID: "hubble-drop-port-forward", + }, + }, } scenario := &types.Scenario{ diff --git a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go b/test/e2e/scenarios/hubble/steps/validate-drop-metric.go index 51d2143f91..969d8db8b5 100644 --- a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go +++ b/test/e2e/scenarios/hubble/steps/validate-drop-metric.go @@ -118,3 +118,7 @@ func (v *ValidateHubbleDropMetric) Prevalidate() error { func (v *ValidateHubbleDropMetric) Postvalidate() error { return nil } + +func (v *ValidateHubbleDropMetric) Stop() error { + return nil +} From 7f6c966a25413c057f9f6a541211a7b4b8abb3e1 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 22 Jan 2024 23:16:53 +0000 Subject: [PATCH 05/11] add prometheus server test --- test/e2e/scenarios/hubble/amametrics_test.go | 85 ++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 test/e2e/scenarios/hubble/amametrics_test.go diff --git a/test/e2e/scenarios/hubble/amametrics_test.go b/test/e2e/scenarios/hubble/amametrics_test.go new file mode 100644 index 0000000000..a7ce8debc3 --- /dev/null +++ b/test/e2e/scenarios/hubble/amametrics_test.go @@ -0,0 +1,85 @@ +//go:build e2e +// +build e2e + +package hubble + +import ( + "context" + "fmt" + "log" + "testing" + + "github.com/Azure/azure-container-networking/test/e2e/framework/types" + "github.com/prometheus/client_golang/api" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" +) + +const promHubbleJob = "hubble-pods" + +var ( + ErrHubbleTargetNotUp = fmt.Errorf("hubble target not up") + ErrNoActiveTargets = fmt.Errorf("no active targets found") +) + +func TestE2EPrometheusTargets(t *testing.T) { + job := types.NewJob("Verify Prometheus targets are up") + runner := types.NewRunner(t, job) + defer runner.Run() + + job.AddStep(&VerifyPrometheusMetrics{ + Address: "http://localhost:9090", + }, nil) +} + +type VerifyPrometheusMetrics struct { + Address string +} + +func (v *VerifyPrometheusMetrics) Run() error { + client, err := api.NewClient(api.Config{ + Address: v.Address, + }) + if err != nil { + return fmt.Errorf("failed to create prometheus client: %w", err) + } + + promapi := promv1.NewAPI(client) + ctx := context.Background() + targets, err := promapi.Targets(ctx) + if err != nil { + return fmt.Errorf("failed to get targets: %w", err) + } + + if len(targets.Active) == 0 { + return fmt.Errorf("no active targets found: %w", ErrNoActiveTargets) + } + + validTarget := &promv1.ActiveTarget{ + ScrapePool: promHubbleJob, + Health: "up", + } + + for i := range targets.Active { + target := &targets.Active[i] + if target.ScrapePool == validTarget.ScrapePool { + if target.Health != validTarget.Health { + return ErrHubbleTargetNotUp + } + break + } + } + + log.Printf("Verified Hubble Prometheus targets are up") + return nil +} + +func (v *VerifyPrometheusMetrics) Prevalidate() error { + return nil +} + +func (v *VerifyPrometheusMetrics) Postvalidate() error { + return nil +} + +func (v *VerifyPrometheusMetrics) Stop() { +} From 94f08f28fc9c8229d8d0094dec7a947c987fc52d Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 26 Jan 2024 01:34:58 +0000 Subject: [PATCH 06/11] validate drop metric --- test/e2e/framework/types/job.go | 10 +- test/e2e/framework/types/step_stop.go | 4 +- test/e2e/scenarios/hubble/scenarios.go | 147 +++++++++++------- .../validate-ama-targets.go} | 20 +-- .../hubble/steps/validate-drop-metric.go | 26 ++-- .../hubble/steps/validate-metrics.go | 24 ++- 6 files changed, 130 insertions(+), 101 deletions(-) rename test/e2e/scenarios/hubble/{amametrics_test.go => steps/validate-ama-targets.go} (76%) diff --git a/test/e2e/framework/types/job.go b/test/e2e/framework/types/job.go index 0ed66606f4..c96ce03010 100644 --- a/test/e2e/framework/types/job.go +++ b/test/e2e/framework/types/job.go @@ -95,7 +95,6 @@ func (j *Job) Run() error { for _, wrapper := range j.Steps { responseDivider(reflect.TypeOf(wrapper.Step).Elem().Name()) - log.Printf("INFO: step options provided: %+v\n", wrapper.Opts) err := wrapper.Step.Run() if wrapper.Opts.ExpectError && err == nil { return fmt.Errorf("expected error from step %s but got nil: %w", reflect.TypeOf(wrapper.Step).Elem().Name(), ErrNilError) @@ -209,13 +208,14 @@ func (j *Job) validateStep(stepw *StepWrapper) error { storedValue := j.Values.Get(parameter) if storedValue == "" { - if value != "" { - - fmt.Printf("%s setting parameter %s in job context to %s\n", stepName, parameter, value) + if stepw.Opts.SkipSavingParamatersToJob { + continue + } else if value != "" { + fmt.Printf("\"%s\" setting parameter \"%s\" in job context to \"%s\"\n", stepName, parameter, value) j.Values.Set(parameter, value) } else { - return fmt.Errorf("missing parameter %s for step %s: %w", parameter, stepName, ErrMissingParameter) + return fmt.Errorf("missing parameter \"%s\" for step \"%s\": %w", parameter, stepName, ErrMissingParameter) } continue } diff --git a/test/e2e/framework/types/step_stop.go b/test/e2e/framework/types/step_stop.go index 9a7ee3f151..9c5390fc88 100644 --- a/test/e2e/framework/types/step_stop.go +++ b/test/e2e/framework/types/step_stop.go @@ -2,6 +2,7 @@ package types import ( "fmt" + "log" "reflect" ) @@ -11,9 +12,10 @@ type Stop struct { } func (c *Stop) Run() error { + stepName := reflect.TypeOf(c.Step).Elem().Name() + log.Println("stopping step:", stepName) err := c.Step.Stop() if err != nil { - stepName := reflect.TypeOf(c.Step).Elem().Name() return fmt.Errorf("failed to stop step: %s with err %w", stepName, err) } return nil diff --git a/test/e2e/scenarios/hubble/scenarios.go b/test/e2e/scenarios/hubble/scenarios.go index ff1379dc82..2a2c080880 100644 --- a/test/e2e/scenarios/hubble/scenarios.go +++ b/test/e2e/scenarios/hubble/scenarios.go @@ -17,74 +17,111 @@ const ( TCP = "TCP" UDP = "UDP" - Delay = 30 * time.Second + Delay = 5 * time.Second ) -func ValidateDropMetric() *types.Scenario { - Steps := []*types.StepWrapper{ - { - Step: &k8s.CreateKapingerDeployment{ - KapingerNamespace: "kube-system", - KapingerReplicas: "1", +func ValidateAMATargets() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &k8s.PortForward{ + Namespace: "kube-system", + LabelSelector: "k8s-app=cilium", + LocalPort: "9965", + RemotePort: "9965", + }, + Opts: &types.StepOptions{ + RunInBackgroundWithID: "validate-ama-targets", + }, }, - }, - { - Step: &k8s.CreateDenyAllNetworkPolicy{ - NetworkPolicyNamespace: "kube-system", - DenyAllLabelSelector: "app=agnhost-a", + { + Step: &steps.VerifyPrometheusMetrics{ + Address: "http://localhost:9090", + }, }, - }, - { - Step: &k8s.CreateAgnhostStatefulSet{ - AgnhostName: "agnhost-a", - AgnhostNamespace: "kube-system", + { + Step: &types.Stop{ + BackgroundID: "validate-ama-targets", + }, }, }, - { - Step: &k8s.ExecInPod{ - PodName: "agnhost-a-0", - PodNamespace: "kube-system", - Command: "curl -s -m 5 bing.com", + } +} + +func ValidateDropMetric() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &k8s.CreateKapingerDeployment{ + KapingerNamespace: "kube-system", + KapingerReplicas: "1", + }, }, - Opts: &types.StepOptions{ - ExpectError: true, + { + Step: &k8s.CreateDenyAllNetworkPolicy{ + NetworkPolicyNamespace: "kube-system", + DenyAllLabelSelector: "app=agnhost-a", + }, }, - }, - { // metrics take some time to show up, so sleep for a bit - Step: &types.Sleep{ - Duration: Delay, + { + Step: &k8s.CreateAgnhostStatefulSet{ + AgnhostName: "agnhost-a", + AgnhostNamespace: "kube-system", + }, }, - }, - { - Step: &k8s.PortForward{ - Namespace: "kube-system", - LabelSelector: "k8s-app=cilium", - LocalPort: "9965", - RemotePort: "9965", - OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace + { + Step: &k8s.ExecInPod{ + PodName: "agnhost-a-0", + PodNamespace: "kube-system", + Command: "curl -s -m 5 bing.com", + }, + Opts: &types.StepOptions{ + ExpectError: true, + SkipSavingParamatersToJob: true, + }, }, - Opts: &types.StepOptions{ - RunInBackgroundWithID: "hubble-drop-port-forward", + { + Step: &types.Sleep{ + Duration: Delay, + }, }, - }, - { - Step: &steps.ValidateHubbleDropMetric{ - PortForwardedHubblePort: "9965", - Source: "agnhost-a", - Reason: PolicyDenied, - Protocol: UDP, + // run curl again + { + Step: &k8s.ExecInPod{ + PodName: "agnhost-a-0", + PodNamespace: "kube-system", + Command: "curl -s -m 5 bing.com", + }, + Opts: &types.StepOptions{ + ExpectError: true, + SkipSavingParamatersToJob: true, + }, }, - }, - { - Step: &types.Stop{ - BackgroundID: "hubble-drop-port-forward", + { + Step: &k8s.PortForward{ + Namespace: "kube-system", + LabelSelector: "k8s-app=cilium", + LocalPort: "9965", + RemotePort: "9965", + OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace + }, + Opts: &types.StepOptions{ + RunInBackgroundWithID: "hubble-drop-port-forward", + }, + }, + { + Step: &steps.ValidateHubbleDropMetric{ + PortForwardedHubblePort: "9965", + Source: "agnhost-a", + Reason: PolicyDenied, + Protocol: UDP, + }, + }, + { + Step: &types.Stop{ + BackgroundID: "hubble-drop-port-forward", + }, }, }, } - - scenario := &types.Scenario{ - Steps: Steps, - } - - return scenario } diff --git a/test/e2e/scenarios/hubble/amametrics_test.go b/test/e2e/scenarios/hubble/steps/validate-ama-targets.go similarity index 76% rename from test/e2e/scenarios/hubble/amametrics_test.go rename to test/e2e/scenarios/hubble/steps/validate-ama-targets.go index a7ce8debc3..cdd34b6c8b 100644 --- a/test/e2e/scenarios/hubble/amametrics_test.go +++ b/test/e2e/scenarios/hubble/steps/validate-ama-targets.go @@ -1,15 +1,10 @@ -//go:build e2e -// +build e2e - -package hubble +package steps import ( "context" "fmt" "log" - "testing" - "github.com/Azure/azure-container-networking/test/e2e/framework/types" "github.com/prometheus/client_golang/api" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" ) @@ -21,16 +16,6 @@ var ( ErrNoActiveTargets = fmt.Errorf("no active targets found") ) -func TestE2EPrometheusTargets(t *testing.T) { - job := types.NewJob("Verify Prometheus targets are up") - runner := types.NewRunner(t, job) - defer runner.Run() - - job.AddStep(&VerifyPrometheusMetrics{ - Address: "http://localhost:9090", - }, nil) -} - type VerifyPrometheusMetrics struct { Address string } @@ -81,5 +66,6 @@ func (v *VerifyPrometheusMetrics) Postvalidate() error { return nil } -func (v *VerifyPrometheusMetrics) Stop() { +func (v *VerifyPrometheusMetrics) Stop() error { + return nil } diff --git a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go b/test/e2e/scenarios/hubble/steps/validate-drop-metric.go index 969d8db8b5..72eaad9186 100644 --- a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go +++ b/test/e2e/scenarios/hubble/steps/validate-drop-metric.go @@ -8,6 +8,7 @@ import ( "net/http" "reflect" + "github.com/Azure/azure-container-networking/test/internal/retry" promclient "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" ) @@ -19,6 +20,8 @@ const ( sourceKey = "source" protcolKey = "protocol" reason = "reason" + + defaultRetryAttempts = 20 ) type ValidateHubbleDropMetric struct { @@ -29,6 +32,8 @@ type ValidateHubbleDropMetric struct { } func (v *ValidateHubbleDropMetric) Run() error { + defaultRetrier := retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} + promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.PortForwardedHubblePort) ctx := context.Background() pctx, cancel := context.WithCancel(ctx) @@ -43,11 +48,16 @@ func (v *ValidateHubbleDropMetric) Run() error { metrics := map[string]*promclient.MetricFamily{} scrapeMetricsFn := func() error { - log.Printf("attempting scrape metrics on %s", promAddress) + log.Printf("checking for drop metrics on %s", promAddress) var err error metrics, err = getPrometheusDropMetrics(promAddress) if err != nil { - return fmt.Errorf("failed to get prometheus metrics: %w", err) + return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeout, err) + } + + err = verifyLabelsPresent(metrics, validMetric) + if err != nil { + return fmt.Errorf("failed to find metric matching %+v: %w", validMetric, ErrNoMetricFound) } return nil @@ -55,18 +65,14 @@ func (v *ValidateHubbleDropMetric) Run() error { err := defaultRetrier.Do(pctx, scrapeMetricsFn) if err != nil { - return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeoutSeconds, err) - } - - if !verifyLabelsPresent(metrics, validMetric) { - return fmt.Errorf("failed to find metric matching %+v: %w", validMetric, ErrNoMetricFound) + return fmt.Errorf("failed to get prometheus metrics: %w", err) } log.Printf("found metric matching %+v\n", validMetric) return nil } -func verifyLabelsPresent(data map[string]*promclient.MetricFamily, validMetric map[string]string) bool { +func verifyLabelsPresent(data map[string]*promclient.MetricFamily, validMetric map[string]string) error { for _, metric := range data { if metric.GetName() == "hubble_drop_total" { for _, metric := range metric.GetMetric() { @@ -77,13 +83,13 @@ func verifyLabelsPresent(data map[string]*promclient.MetricFamily, validMetric m metricLabels[label.GetName()] = label.GetValue() } if reflect.DeepEqual(metricLabels, validMetric) { - return true + return nil } } } } - return false + return fmt.Errorf("failed to find metric matching %+v: %w", validMetric, ErrNoMetricFound) } func getPrometheusDropMetrics(url string) (map[string]*promclient.MetricFamily, error) { diff --git a/test/e2e/scenarios/hubble/steps/validate-metrics.go b/test/e2e/scenarios/hubble/steps/validate-metrics.go index 6ea4d9db20..14617ecf2a 100644 --- a/test/e2e/scenarios/hubble/steps/validate-metrics.go +++ b/test/e2e/scenarios/hubble/steps/validate-metrics.go @@ -13,25 +13,23 @@ import ( ) const ( - defaultTimeoutSeconds = 300 - defaultRetryDelay = 5 * time.Second - defaultRetryAttempts = 60 -) - -var ( - requiredMetrics = []string{ - "hubble_flows_processed_total", - "hubble_tcp_flags_total", - } - defaultRetrier = retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} + // eventually should be scoped per test, but avoiding magic number complaints for the moment + defaultTimeout = 300 * time.Second + defaultRetryDelay = 5 * time.Second ) +var requiredMetrics = []string{ + "hubble_flows_processed_total", + "hubble_tcp_flags_total", +} + type ValidateHubbleFlowMetric struct { LocalPort string } func (v *ValidateHubbleFlowMetric) Run() error { + defaultRetrier := retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.LocalPort) log.Printf("require all metrics to be present: %+v\n", requiredMetrics) ctx := context.Background() @@ -47,11 +45,11 @@ func (v *ValidateHubbleFlowMetric) Run() error { return nil } - portForwardCtx, cancel := context.WithTimeout(ctx, defaultTimeoutSeconds*time.Second) + portForwardCtx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() if err := defaultRetrier.Do(portForwardCtx, scrapeMetricsFn); err != nil { - return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeoutSeconds, err) + return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeout, err) } for _, reqMetric := range requiredMetrics { From f492e63aae41bf0a691d0e2c1d58dcc2a5d36315 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 26 Jan 2024 02:00:31 +0000 Subject: [PATCH 07/11] reorg scenarios --- .../framework/azure/create-cilium-cluster.go | 4 -- test/e2e/framework/azure/create-cluster.go | 6 +-- test/e2e/framework/azure/create-rg.go | 4 -- test/e2e/framework/azure/create-vnet.go | 8 ---- test/e2e/framework/azure/delete-cluster.go | 4 -- test/e2e/framework/azure/delete-rg.go | 12 +----- test/e2e/framework/azure/enable-ama.go | 4 -- test/e2e/framework/azure/get-kubeconfig.go | 4 -- .../kubernetes/create-agnhost-statefulset.go | 4 -- .../kubernetes/create-kapinger-deployment.go | 4 -- .../kubernetes/create-network-policy.go | 8 ---- test/e2e/framework/kubernetes/exec-pod.go | 4 -- test/e2e/framework/kubernetes/port-forward.go | 5 --- .../types/examples/background_test.go | 4 -- test/e2e/framework/types/job.go | 18 ++++----- test/e2e/framework/types/runner.go | 2 + test/e2e/framework/types/step.go | 17 +++++++- test/e2e/framework/types/step_sleep.go | 4 -- test/e2e/framework/types/step_stop.go | 4 -- test/e2e/scenarios/cns/scenarios.go | 3 -- .../scenarios/hubble/azuremonitor/scenario.go | 35 +++++++++++++++++ .../validate-ama-targets.go | 6 +-- .../hubble/{scenarios.go => drop/scenario.go} | 39 +++---------------- .../{steps => drop}/validate-drop-metric.go | 6 +-- test/e2e/scenarios/hubble/flow/scenario.go | 16 ++++++++ .../{steps => flow}/validate-metrics.go | 11 +++--- ...arios_test.go => hubble_scenarios_test.go} | 5 ++- .../networkobservability/hubble_test.go | 2 +- 28 files changed, 94 insertions(+), 149 deletions(-) delete mode 100644 test/e2e/scenarios/cns/scenarios.go create mode 100644 test/e2e/scenarios/hubble/azuremonitor/scenario.go rename test/e2e/scenarios/hubble/{steps => azuremonitor}/validate-ama-targets.go (94%) rename test/e2e/scenarios/hubble/{scenarios.go => drop/scenario.go} (74%) rename test/e2e/scenarios/hubble/{steps => drop}/validate-drop-metric.go (97%) create mode 100644 test/e2e/scenarios/hubble/flow/scenario.go rename test/e2e/scenarios/hubble/{steps => flow}/validate-metrics.go (92%) rename test/e2e/scenarios/hubble/{scenarios_test.go => hubble_scenarios_test.go} (90%) diff --git a/test/e2e/framework/azure/create-cilium-cluster.go b/test/e2e/framework/azure/create-cilium-cluster.go index e4a8320a7b..a31b47081d 100644 --- a/test/e2e/framework/azure/create-cilium-cluster.go +++ b/test/e2e/framework/azure/create-cilium-cluster.go @@ -68,10 +68,6 @@ func (c *CreateBYOCiliumCluster) Prevalidate() error { return nil } -func (c *CreateBYOCiliumCluster) Postvalidate() error { - return nil -} - func (c *CreateBYOCiliumCluster) Stop() error { return nil } diff --git a/test/e2e/framework/azure/create-cluster.go b/test/e2e/framework/azure/create-cluster.go index 3226c3a44f..f21a02ec99 100644 --- a/test/e2e/framework/azure/create-cluster.go +++ b/test/e2e/framework/azure/create-cluster.go @@ -13,7 +13,7 @@ import ( const ( MaxNumberOfNodes = 3 MaxPodsPerNode = 250 - AgentSKU = "Standard_D4s_v3" + AgentSKU = "Standard_DS4_v2" ) var defaultClusterCreateTimeout = 30 * time.Minute @@ -110,10 +110,6 @@ func (c *CreateCluster) Prevalidate() error { return nil } -func (c *CreateCluster) Postvalidate() error { - return nil -} - func (c *CreateCluster) Stop() error { return nil } diff --git a/test/e2e/framework/azure/create-rg.go b/test/e2e/framework/azure/create-rg.go index d4de2775af..190d67ec13 100644 --- a/test/e2e/framework/azure/create-rg.go +++ b/test/e2e/framework/azure/create-rg.go @@ -43,10 +43,6 @@ func (c *CreateResourceGroup) Prevalidate() error { return nil } -func (c *CreateResourceGroup) Postvalidate() error { - return nil -} - func (c *CreateResourceGroup) Stop() error { return nil } diff --git a/test/e2e/framework/azure/create-vnet.go b/test/e2e/framework/azure/create-vnet.go index a1ade96e53..40f414bc23 100644 --- a/test/e2e/framework/azure/create-vnet.go +++ b/test/e2e/framework/azure/create-vnet.go @@ -59,10 +59,6 @@ func (c *CreateVNet) Prevalidate() error { return nil } -func (c *CreateVNet) Postvalidate() error { - return nil -} - func (c *CreateVNet) Stop() error { return nil } @@ -109,10 +105,6 @@ func (c *CreateSubnet) Prevalidate() error { return nil } -func (c *CreateSubnet) Postvalidate() error { - return nil -} - func (c *CreateSubnet) Stop() error { return nil } diff --git a/test/e2e/framework/azure/delete-cluster.go b/test/e2e/framework/azure/delete-cluster.go index a3febc60bd..ebb7b11043 100644 --- a/test/e2e/framework/azure/delete-cluster.go +++ b/test/e2e/framework/azure/delete-cluster.go @@ -43,10 +43,6 @@ func (d *DeleteCluster) Prevalidate() error { return nil } -func (d *DeleteCluster) Postvalidate() error { - return nil -} - func (d *DeleteCluster) Stop() error { return nil } diff --git a/test/e2e/framework/azure/delete-rg.go b/test/e2e/framework/azure/delete-rg.go index fbeb8b4da6..9691f0fe4e 100644 --- a/test/e2e/framework/azure/delete-rg.go +++ b/test/e2e/framework/azure/delete-rg.go @@ -27,18 +27,12 @@ func (d *DeleteResourceGroup) Run() error { if err != nil { return fmt.Errorf("failed to create resource group client: %w", err) } - forceDeleteType := "Microsoft.Compute/virtualMachines,Microsoft.Compute/virtualMachineScaleSets" - poller, err := clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)}) + _, err = clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)}) if err != nil { return fmt.Errorf("failed to finish the delete resource group request: %w", err) } - _, err = poller.PollUntilDone(ctx, nil) - if err != nil { - return fmt.Errorf("failed to pull the result for delete resource group: %w", err) - } - log.Printf("resource group \"%s\" deleted successfully", d.ResourceGroupName) return nil } @@ -47,10 +41,6 @@ func (d *DeleteResourceGroup) Prevalidate() error { return nil } -func (d *DeleteResourceGroup) Postvalidate() error { - return nil -} - func (d *DeleteResourceGroup) Stop() error { return nil } diff --git a/test/e2e/framework/azure/enable-ama.go b/test/e2e/framework/azure/enable-ama.go index f8346c4039..0df02e9e8d 100644 --- a/test/e2e/framework/azure/enable-ama.go +++ b/test/e2e/framework/azure/enable-ama.go @@ -112,10 +112,6 @@ func (c *CreateAzureMonitor) Prevalidate() error { return nil } -func (c *CreateAzureMonitor) Postvalidate() error { - return nil -} - func (c *CreateAzureMonitor) Stop() error { return nil } diff --git a/test/e2e/framework/azure/get-kubeconfig.go b/test/e2e/framework/azure/get-kubeconfig.go index 7651f47a5b..f25cddd9b0 100644 --- a/test/e2e/framework/azure/get-kubeconfig.go +++ b/test/e2e/framework/azure/get-kubeconfig.go @@ -48,10 +48,6 @@ func (c *GetAKSKubeConfig) Prevalidate() error { return nil } -func (c *GetAKSKubeConfig) Postvalidate() error { - return nil -} - func (c *GetAKSKubeConfig) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go index f86223630b..b35649c9d7 100644 --- a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go +++ b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go @@ -66,10 +66,6 @@ func (c *CreateAgnhostStatefulSet) Prevalidate() error { return nil } -func (c *CreateAgnhostStatefulSet) Postvalidate() error { - return nil -} - func (c *CreateAgnhostStatefulSet) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/create-kapinger-deployment.go b/test/e2e/framework/kubernetes/create-kapinger-deployment.go index abe8f7a200..4cab47068a 100644 --- a/test/e2e/framework/kubernetes/create-kapinger-deployment.go +++ b/test/e2e/framework/kubernetes/create-kapinger-deployment.go @@ -70,10 +70,6 @@ func (c *CreateKapingerDeployment) Prevalidate() error { return nil } -func (c *CreateKapingerDeployment) Postvalidate() error { - return nil -} - func (c *CreateKapingerDeployment) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/create-network-policy.go b/test/e2e/framework/kubernetes/create-network-policy.go index 900cc45f47..2b67210c3e 100644 --- a/test/e2e/framework/kubernetes/create-network-policy.go +++ b/test/e2e/framework/kubernetes/create-network-policy.go @@ -72,10 +72,6 @@ func (c *CreateDenyAllNetworkPolicy) Prevalidate() error { return nil } -func (c *CreateDenyAllNetworkPolicy) Postvalidate() error { - return nil -} - func (c *CreateDenyAllNetworkPolicy) Stop() error { return nil } @@ -112,7 +108,3 @@ func (d *DeleteDenyAllNetworkPolicy) Run() error { func (d *DeleteDenyAllNetworkPolicy) Prevalidate() error { return nil } - -func (d *DeleteDenyAllNetworkPolicy) Postvalidate() error { - return nil -} diff --git a/test/e2e/framework/kubernetes/exec-pod.go b/test/e2e/framework/kubernetes/exec-pod.go index 1ce916a7c0..25fa6be12c 100644 --- a/test/e2e/framework/kubernetes/exec-pod.go +++ b/test/e2e/framework/kubernetes/exec-pod.go @@ -39,10 +39,6 @@ func (e *ExecInPod) Prevalidate() error { return nil } -func (e *ExecInPod) Postvalidate() error { - return nil -} - func (e *ExecInPod) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/port-forward.go b/test/e2e/framework/kubernetes/port-forward.go index 423eb0f4a1..32cb08be63 100644 --- a/test/e2e/framework/kubernetes/port-forward.go +++ b/test/e2e/framework/kubernetes/port-forward.go @@ -156,11 +156,6 @@ func (p *PortForward) Prevalidate() error { return nil } -func (p *PortForward) Postvalidate() error { - p.pf.Stop() - return nil -} - func (p *PortForward) Stop() error { p.pf.Stop() return nil diff --git a/test/e2e/framework/types/examples/background_test.go b/test/e2e/framework/types/examples/background_test.go index 3abc64a3ef..b1c63e07c0 100644 --- a/test/e2e/framework/types/examples/background_test.go +++ b/test/e2e/framework/types/examples/background_test.go @@ -60,10 +60,6 @@ func (t *TestBackground) Prevalidate() error { return nil } -func (t *TestBackground) Postvalidate() error { - return nil -} - type counter struct { ticker *time.Ticker count int diff --git a/test/e2e/framework/types/job.go b/test/e2e/framework/types/job.go index c96ce03010..edd10fc75b 100644 --- a/test/e2e/framework/types/job.go +++ b/test/e2e/framework/types/job.go @@ -31,7 +31,8 @@ type StepWrapper struct { Opts *StepOptions } -// A Scenario is a logical grouping of steps +// A Scenario is a logical grouping of steps, used to describe a scenario such as "test drop metrics" +// which will require port forwarding, exec'ing, scraping, etc. type Scenario struct { Steps []*StepWrapper } @@ -103,12 +104,6 @@ func (j *Job) Run() error { } } - for _, wrapper := range j.Steps { - err := wrapper.Step.Postvalidate() - if err != nil { - return err //nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself - } - } return nil } @@ -208,13 +203,14 @@ func (j *Job) validateStep(stepw *StepWrapper) error { storedValue := j.Values.Get(parameter) if storedValue == "" { - if stepw.Opts.SkipSavingParamatersToJob { + + switch { + case stepw.Opts.SkipSavingParamatersToJob: continue - } else if value != "" { + case value != "": fmt.Printf("\"%s\" setting parameter \"%s\" in job context to \"%s\"\n", stepName, parameter, value) j.Values.Set(parameter, value) - - } else { + default: return fmt.Errorf("missing parameter \"%s\" for step \"%s\": %w", parameter, stepName, ErrMissingParameter) } continue diff --git a/test/e2e/framework/types/runner.go b/test/e2e/framework/types/runner.go index 76cc3984fc..835fe05f32 100644 --- a/test/e2e/framework/types/runner.go +++ b/test/e2e/framework/types/runner.go @@ -6,6 +6,8 @@ import ( "github.com/stretchr/testify/require" ) +// A wrapper around a job, so that internal job components don't require things like *testing.T +// and can be reused elsewhere type Runner struct { t *testing.T Job *Job diff --git a/test/e2e/framework/types/step.go b/test/e2e/framework/types/step.go index fc82cb1ae2..dd14f2bcd1 100644 --- a/test/e2e/framework/types/step.go +++ b/test/e2e/framework/types/step.go @@ -1,14 +1,27 @@ package types var DefaultOpts = StepOptions{ - ExpectError: false, + // when wanting to expect an error, set to true + ExpectError: false, + + // when wanting to avoid saving the parameters to the job, + // such as a repetetive task where step is used multiple times sequentially, + // but parameters are different each time SkipSavingParamatersToJob: false, } type Step interface { + // Useful when wanting to do parameter checking, for example + // if a parameter length is known to be required less than 80 characters, + // do this here so we don't find out later on when we run the step + // when possible, try to avoid making external calls, this should be fast and simple Prevalidate() error + + // Primary step where test logic is executed + // Returning an error will cause the test to fail Run() error - Postvalidate() error + + // Require for background steps Stop() error } diff --git a/test/e2e/framework/types/step_sleep.go b/test/e2e/framework/types/step_sleep.go index 1751fe7367..a1a9b4a627 100644 --- a/test/e2e/framework/types/step_sleep.go +++ b/test/e2e/framework/types/step_sleep.go @@ -22,7 +22,3 @@ func (c *Sleep) Stop() error { func (c *Sleep) Prevalidate() error { return nil } - -func (c *Sleep) Postvalidate() error { - return nil -} diff --git a/test/e2e/framework/types/step_stop.go b/test/e2e/framework/types/step_stop.go index 9c5390fc88..226fd8b2f5 100644 --- a/test/e2e/framework/types/step_stop.go +++ b/test/e2e/framework/types/step_stop.go @@ -28,7 +28,3 @@ func (c *Stop) Stop() error { func (c *Stop) Prevalidate() error { return nil } - -func (c *Stop) Postvalidate() error { - return nil -} diff --git a/test/e2e/scenarios/cns/scenarios.go b/test/e2e/scenarios/cns/scenarios.go deleted file mode 100644 index 8e4e36f7ca..0000000000 --- a/test/e2e/scenarios/cns/scenarios.go +++ /dev/null @@ -1,3 +0,0 @@ -package cns - -// reserved diff --git a/test/e2e/scenarios/hubble/azuremonitor/scenario.go b/test/e2e/scenarios/hubble/azuremonitor/scenario.go new file mode 100644 index 0000000000..889a3ed217 --- /dev/null +++ b/test/e2e/scenarios/hubble/azuremonitor/scenario.go @@ -0,0 +1,35 @@ +package azuremonitor + +import ( + k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" +) + +// todo: once AMA is rolled out +func ValidateAMATargets() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &k8s.PortForward{ + Namespace: "kube-system", + LabelSelector: "k8s-app=cilium", + LocalPort: "9965", + RemotePort: "9965", + }, + Opts: &types.StepOptions{ + RunInBackgroundWithID: "validate-ama-targets", + }, + }, + { + Step: &VerifyPrometheusMetrics{ + Address: "http://localhost:9090", + }, + }, + { + Step: &types.Stop{ + BackgroundID: "validate-ama-targets", + }, + }, + }, + } +} diff --git a/test/e2e/scenarios/hubble/steps/validate-ama-targets.go b/test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go similarity index 94% rename from test/e2e/scenarios/hubble/steps/validate-ama-targets.go rename to test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go index cdd34b6c8b..42af278598 100644 --- a/test/e2e/scenarios/hubble/steps/validate-ama-targets.go +++ b/test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go @@ -1,4 +1,4 @@ -package steps +package azuremonitor import ( "context" @@ -62,10 +62,6 @@ func (v *VerifyPrometheusMetrics) Prevalidate() error { return nil } -func (v *VerifyPrometheusMetrics) Postvalidate() error { - return nil -} - func (v *VerifyPrometheusMetrics) Stop() error { return nil } diff --git a/test/e2e/scenarios/hubble/scenarios.go b/test/e2e/scenarios/hubble/drop/scenario.go similarity index 74% rename from test/e2e/scenarios/hubble/scenarios.go rename to test/e2e/scenarios/hubble/drop/scenario.go index 2a2c080880..8911c4a2b0 100644 --- a/test/e2e/scenarios/hubble/scenarios.go +++ b/test/e2e/scenarios/hubble/drop/scenario.go @@ -1,11 +1,10 @@ -package hubble +package drop import ( "time" k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" "github.com/Azure/azure-container-networking/test/e2e/framework/types" - "github.com/Azure/azure-container-networking/test/e2e/scenarios/hubble/steps" ) const ( @@ -17,37 +16,11 @@ const ( TCP = "TCP" UDP = "UDP" - Delay = 5 * time.Second + sleepDelay = 5 + defaultTimeout = 300 * time.Second + defaultRetryDelay = 5 * time.Second ) -func ValidateAMATargets() *types.Scenario { - return &types.Scenario{ - Steps: []*types.StepWrapper{ - { - Step: &k8s.PortForward{ - Namespace: "kube-system", - LabelSelector: "k8s-app=cilium", - LocalPort: "9965", - RemotePort: "9965", - }, - Opts: &types.StepOptions{ - RunInBackgroundWithID: "validate-ama-targets", - }, - }, - { - Step: &steps.VerifyPrometheusMetrics{ - Address: "http://localhost:9090", - }, - }, - { - Step: &types.Stop{ - BackgroundID: "validate-ama-targets", - }, - }, - }, - } -} - func ValidateDropMetric() *types.Scenario { return &types.Scenario{ Steps: []*types.StepWrapper{ @@ -82,7 +55,7 @@ func ValidateDropMetric() *types.Scenario { }, { Step: &types.Sleep{ - Duration: Delay, + Duration: sleepDelay, }, }, // run curl again @@ -110,7 +83,7 @@ func ValidateDropMetric() *types.Scenario { }, }, { - Step: &steps.ValidateHubbleDropMetric{ + Step: &ValidateHubbleDropMetric{ PortForwardedHubblePort: "9965", Source: "agnhost-a", Reason: PolicyDenied, diff --git a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go b/test/e2e/scenarios/hubble/drop/validate-drop-metric.go similarity index 97% rename from test/e2e/scenarios/hubble/steps/validate-drop-metric.go rename to test/e2e/scenarios/hubble/drop/validate-drop-metric.go index 72eaad9186..bb856c4ace 100644 --- a/test/e2e/scenarios/hubble/steps/validate-drop-metric.go +++ b/test/e2e/scenarios/hubble/drop/validate-drop-metric.go @@ -1,4 +1,4 @@ -package steps +package drop import ( "context" @@ -121,10 +121,6 @@ func (v *ValidateHubbleDropMetric) Prevalidate() error { return nil } -func (v *ValidateHubbleDropMetric) Postvalidate() error { - return nil -} - func (v *ValidateHubbleDropMetric) Stop() error { return nil } diff --git a/test/e2e/scenarios/hubble/flow/scenario.go b/test/e2e/scenarios/hubble/flow/scenario.go new file mode 100644 index 0000000000..b89a056ced --- /dev/null +++ b/test/e2e/scenarios/hubble/flow/scenario.go @@ -0,0 +1,16 @@ +package flow + +import "github.com/Azure/azure-container-networking/test/e2e/framework/types" + +// todo: once AMA is rolled out +func ValidateAMATargets() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &ValidateHubbleFlowMetric{ + LocalPort: "9090", + }, + }, + }, + } +} diff --git a/test/e2e/scenarios/hubble/steps/validate-metrics.go b/test/e2e/scenarios/hubble/flow/validate-metrics.go similarity index 92% rename from test/e2e/scenarios/hubble/steps/validate-metrics.go rename to test/e2e/scenarios/hubble/flow/validate-metrics.go index 14617ecf2a..4c28c84bbe 100644 --- a/test/e2e/scenarios/hubble/steps/validate-metrics.go +++ b/test/e2e/scenarios/hubble/flow/validate-metrics.go @@ -1,4 +1,4 @@ -package steps +package flow import ( "context" @@ -13,10 +13,9 @@ import ( ) const ( - - // eventually should be scoped per test, but avoiding magic number complaints for the moment - defaultTimeout = 300 * time.Second - defaultRetryDelay = 5 * time.Second + defaultRetryAttempts = 20 + defaultTimeout = 300 * time.Second + defaultRetryDelay = 5 * time.Second ) var requiredMetrics = []string{ @@ -67,7 +66,7 @@ func (v *ValidateHubbleFlowMetric) Prevalidate() error { return nil } -func (v *ValidateHubbleFlowMetric) Postvalidate() error { +func (v *ValidateHubbleFlowMetric) Stop() error { return nil } diff --git a/test/e2e/scenarios/hubble/scenarios_test.go b/test/e2e/scenarios/hubble/hubble_scenarios_test.go similarity index 90% rename from test/e2e/scenarios/hubble/scenarios_test.go rename to test/e2e/scenarios/hubble/hubble_scenarios_test.go index 28f873ca63..cea8e0f8de 100644 --- a/test/e2e/scenarios/hubble/scenarios_test.go +++ b/test/e2e/scenarios/hubble/hubble_scenarios_test.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/test/e2e/framework/azure" "github.com/Azure/azure-container-networking/test/e2e/framework/types" + "github.com/Azure/azure-container-networking/test/e2e/scenarios/hubble/drop" ) const ( @@ -32,7 +33,7 @@ func TestE2EDropHubbleMetrics(t *testing.T) { job.AddStep(&azure.CreateResourceGroup{ SubscriptionID: sub, ResourceGroupName: testName, - Location: "westus2", + Location: "eastus", }, nil) job.AddStep(&azure.CreateVNet{ @@ -56,7 +57,7 @@ func TestE2EDropHubbleMetrics(t *testing.T) { KubeConfigFilePath: "./test.pem", }, nil) - job.AddScenario(ValidateDropMetric()) + job.AddScenario(drop.ValidateDropMetric()) job.AddStep(&azure.DeleteResourceGroup{}, nil) } diff --git a/test/integration/networkobservability/hubble_test.go b/test/integration/networkobservability/hubble_test.go index cae832368a..a363b9a97c 100644 --- a/test/integration/networkobservability/hubble_test.go +++ b/test/integration/networkobservability/hubble_test.go @@ -236,7 +236,7 @@ func TestEndpoints(t *testing.T) { portForwardFn := func() error { t.Logf("attempting port forward to a pod with label %s, in namespace %s...", labelSelector, namespace) - if err = pf.ForwardRandomPodWithOptsLabelSelector(portForwardCtx); err != nil { + if err = pf.Forward(portForwardCtx); err != nil { return fmt.Errorf("could not start port forward: %w", err) } return nil From 0891e9b35c248dcb2b7b02cc60543cbe800fa3b5 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 26 Jan 2024 16:47:14 +0000 Subject: [PATCH 08/11] skip e2e in ut --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bdfe4c9947..dc85d2b2a2 100644 --- a/Makefile +++ b/Makefile @@ -775,13 +775,13 @@ CNI_TYPE ?= cilium test-all: ## run all unit tests. @$(eval COVER_FILTER=`go list --tags ignore_uncovered,ignore_autogenerated $(COVER_PKG)/... | tr '\n' ','`) @echo Test coverpkg: $(COVER_FILTER) - go test -mod=readonly -buildvcs=false -tags "unit" -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/... + go test -mod=readonly -buildvcs=false -tags "unit" --skip 'TestE2E*' -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/... test-integration: ## run all integration tests. AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \ CNI_VERSION=$(CNI_VERSION) \ CNS_VERSION=$(CNS_VERSION) \ - go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration... + go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration --skip 'TestE2E*' ./test/integration... test-load: ## run all load tests AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \ @@ -790,7 +790,7 @@ test-load: ## run all load tests go test -timeout 30m -race -tags=load ./test/integration/load... -v test-validate-state: - cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load -run ^TestValidateState + cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load --skip 'TestE2E*' -run ^TestValidateState cd ../../.. test-cyclonus: ## run the cyclonus test for npm. From 943e0c7ed94ac6e5a96190b6042a143fab3e9803 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 26 Jan 2024 17:55:01 +0000 Subject: [PATCH 09/11] test logging --- test/e2e/framework/kubernetes/port-forward.go | 4 +++- test/e2e/framework/types/step_sleep.go | 2 +- test/e2e/scenarios/hubble/drop/scenario.go | 6 ------ test/e2e/scenarios/hubble/drop/validate-drop-metric.go | 6 +++++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/e2e/framework/kubernetes/port-forward.go b/test/e2e/framework/kubernetes/port-forward.go index 32cb08be63..05a5f2afe7 100644 --- a/test/e2e/framework/kubernetes/port-forward.go +++ b/test/e2e/framework/kubernetes/port-forward.go @@ -64,6 +64,7 @@ func (p *PortForward) Run() error { targetPodName := "" if p.OptionalLabelAffinity != "" { // get all pods with label + log.Printf("attempting to find pod with label \"%s\", on a node with a pod with label \"%s\"\n", p.LabelSelector, p.OptionalLabelAffinity) targetPodName, err = p.findPodsWithAffinity(pctx, clientset) if err != nil { return fmt.Errorf("could not find pod with affinity: %w", err) @@ -71,7 +72,6 @@ func (p *PortForward) Run() error { } portForwardFn := func() error { - log.Printf("attempting port forward to a pod with label \"%s\", in namespace \"%s\"...\n", p.LabelSelector, p.Namespace) // if we have a pod name (likely from affinity above), use it, otherwise use label selector opts := k8s.PortForwardingOpts{ @@ -85,6 +85,8 @@ func (p *PortForward) Run() error { opts.PodName = targetPodName } + log.Printf("attempting port forward to pod name \"%s\" with label \"%s\", in namespace \"%s\"...\n", targetPodName, p.LabelSelector, p.Namespace) + p.pf, err = k8s.NewPortForwarder(config, &logger{}, opts) if err != nil { return fmt.Errorf("could not create port forwarder: %w", err) diff --git a/test/e2e/framework/types/step_sleep.go b/test/e2e/framework/types/step_sleep.go index a1a9b4a627..2c89d1deb9 100644 --- a/test/e2e/framework/types/step_sleep.go +++ b/test/e2e/framework/types/step_sleep.go @@ -10,7 +10,7 @@ type Sleep struct { } func (c *Sleep) Run() error { - log.Printf("sleeping for %s...\n", c.Duration) + log.Printf("sleeping for %s...\n", c.Duration.String()) time.Sleep(c.Duration) return nil } diff --git a/test/e2e/scenarios/hubble/drop/scenario.go b/test/e2e/scenarios/hubble/drop/scenario.go index 8911c4a2b0..3efd78af04 100644 --- a/test/e2e/scenarios/hubble/drop/scenario.go +++ b/test/e2e/scenarios/hubble/drop/scenario.go @@ -1,8 +1,6 @@ package drop import ( - "time" - k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" "github.com/Azure/azure-container-networking/test/e2e/framework/types" ) @@ -15,10 +13,6 @@ const ( // L4 protocols TCP = "TCP" UDP = "UDP" - - sleepDelay = 5 - defaultTimeout = 300 * time.Second - defaultRetryDelay = 5 * time.Second ) func ValidateDropMetric() *types.Scenario { diff --git a/test/e2e/scenarios/hubble/drop/validate-drop-metric.go b/test/e2e/scenarios/hubble/drop/validate-drop-metric.go index bb856c4ace..e6d7aa6648 100644 --- a/test/e2e/scenarios/hubble/drop/validate-drop-metric.go +++ b/test/e2e/scenarios/hubble/drop/validate-drop-metric.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "reflect" + "time" "github.com/Azure/azure-container-networking/test/internal/retry" promclient "github.com/prometheus/client_model/go" @@ -21,7 +22,10 @@ const ( protcolKey = "protocol" reason = "reason" - defaultRetryAttempts = 20 + sleepDelay = 5 * time.Second + defaultTimeout = 300 * time.Second + defaultRetryDelay = 5 * time.Second + defaultRetryAttempts = 60 ) type ValidateHubbleDropMetric struct { From e2f45f4028541e505b4793d7d70b0e1967ac37db Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 29 Jan 2024 16:36:55 +0000 Subject: [PATCH 10/11] pod name empty --- test/integration/portforward.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/portforward.go b/test/integration/portforward.go index 90c44ee211..4765b9a715 100644 --- a/test/integration/portforward.go +++ b/test/integration/portforward.go @@ -75,7 +75,7 @@ func NewPortForwarder(restConfig *rest.Config, logger logger, opts PortForwardin func (p *PortForwarder) Forward(ctx context.Context) error { var podName string - if p.opts.PodName != "" { + if p.opts.PodName == "" { pods, err := p.clientset.CoreV1().Pods(p.opts.Namespace).List(ctx, metav1.ListOptions{LabelSelector: p.opts.LabelSelector, FieldSelector: "status.phase=Running"}) if err != nil { return fmt.Errorf("could not list pods in %q with label %q: %w", p.opts.Namespace, p.opts.LabelSelector, err) From b44781e49091dc7141c6f8922b1f20a15183879a Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 29 Jan 2024 18:55:48 +0000 Subject: [PATCH 11/11] wait cleanup --- .pipelines/pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 8603f749bb..460ebd0384 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -596,6 +596,7 @@ stages: - aks_swift_e2e - cilium_e2e - cilium_overlay_e2e + - cilium_h_overlay_e2e - aks_ubuntu_22_linux_e2e - aks_windows_22_e2e - dualstackoverlay_e2e