ci: E2E Framework [Scenarios] [5/6]

.pipelines/networkobservability/e2e.yaml

@@ -0,0 +1,29 @@
+pr: none
+
+stages:
+  - stage: Run_E2E
+    displayName: Run E2E
+    jobs:
+      - job: Run_E2E
+        displayName: Run Network Observability E2E
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+
+        steps:
+          - task: AzureCLI@2
+            displayName: "Verify E2E Manifests"
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptType: "bash"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                go test -v ./test/e2e/manifests/*.go
+
+          - task: AzureCLI@2
+            displayName: "Run Network Observability E2E"
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptType: "bash"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                go test -v ./test/e2e/scenarios/hubble/*.go -timeout 30m

.pipelines/pipeline.yaml

@@ -596,6 +596,7 @@ stages:
         - aks_swift_e2e
         - cilium_e2e
         - cilium_overlay_e2e
+        - cilium_h_overlay_e2e
         - aks_ubuntu_22_linux_e2e
         - aks_windows_22_e2e
         - dualstackoverlay_e2e

Makefile

@@ -775,13 +775,13 @@ CNI_TYPE ?= cilium
 test-all: ## run all unit tests.
 	@$(eval COVER_FILTER=`go list --tags ignore_uncovered,ignore_autogenerated $(COVER_PKG)/... | tr '\n' ','`)
 	@echo Test coverpkg: $(COVER_FILTER)
-	go test -mod=readonly -buildvcs=false -tags "unit" -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/...
+	go test -mod=readonly -buildvcs=false -tags "unit" --skip 'TestE2E*' -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/...
 
 test-integration: ## run all integration tests.
 	AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \
 		CNI_VERSION=$(CNI_VERSION) \
 		CNS_VERSION=$(CNS_VERSION) \
-		go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
+		go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration --skip 'TestE2E*' ./test/integration...
 
 test-load: ## run all load tests
 	AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \
@@ -790,7 +790,7 @@ test-load: ## run all load tests
 		go test -timeout 30m -race -tags=load ./test/integration/load... -v
 
 test-validate-state:
-	cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load -run ^TestValidateState
+	cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load --skip 'TestE2E*' -run ^TestValidateState
 	cd ../../..
 
 test-cyclonus: ## run the cyclonus test for npm.

test/e2e/framework/azure/create-cilium-cluster.go

@@ -68,7 +68,7 @@ func (c *CreateBYOCiliumCluster) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateBYOCiliumCluster) Postvalidate() error {
+func (c *CreateBYOCiliumCluster) Stop() error {
 	return nil
 }
 
@@ -112,6 +112,8 @@ func (c *CreateBYOCiliumCluster) Run() error {
 		return fmt.Errorf("failed to create az client: %w", err)
 	}
 
+	log.Printf("when the cluster is ready, use the below command to access and debug")
+	log.Printf("az aks get-credentials --resource-group %s --name %s --subscription %s", c.ResourceGroupName, c.ClusterName, c.SubscriptionID)
 	log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName)
 
 	poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil)

test/e2e/framework/azure/create-cluster.go

@@ -13,7 +13,7 @@ import (
 const (
 	MaxNumberOfNodes = 3
 	MaxPodsPerNode   = 250
-	AgentSKU         = "Standard_D4s_v3"
+	AgentSKU         = "Standard_DS4_v2"
 )
 
 var defaultClusterCreateTimeout = 30 * time.Minute
@@ -110,6 +110,6 @@ func (c *CreateCluster) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateCluster) Postvalidate() error {
+func (c *CreateCluster) Stop() error {
 	return nil
 }

test/e2e/framework/azure/create-rg.go

@@ -43,6 +43,6 @@ func (c *CreateResourceGroup) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateResourceGroup) Postvalidate() error {
+func (c *CreateResourceGroup) Stop() error {
 	return nil
 }

test/e2e/framework/azure/create-vnet.go

@@ -59,7 +59,7 @@ func (c *CreateVNet) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateVNet) Postvalidate() error {
+func (c *CreateVNet) Stop() error {
 	return nil
 }
 
@@ -105,6 +105,6 @@ func (c *CreateSubnet) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateSubnet) Postvalidate() error {
+func (c *CreateSubnet) Stop() error {
 	return nil
 }

test/e2e/framework/azure/delete-cluster.go

@@ -43,6 +43,6 @@ func (d *DeleteCluster) Prevalidate() error {
 	return nil
 }
 
-func (d *DeleteCluster) Postvalidate() error {
+func (d *DeleteCluster) Stop() error {
 	return nil
 }

test/e2e/framework/azure/delete-rg.go

@@ -27,18 +27,12 @@ func (d *DeleteResourceGroup) Run() error {
 	if err != nil {
 		return fmt.Errorf("failed to create resource group client: %w", err)
 	}
-
 	forceDeleteType := "Microsoft.Compute/virtualMachines,Microsoft.Compute/virtualMachineScaleSets"
-	poller, err := clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)})
+	_, err = clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)})
 	if err != nil {
 		return fmt.Errorf("failed to finish the delete resource group request: %w", err)
 	}
 
-	_, err = poller.PollUntilDone(ctx, nil)
-	if err != nil {
-		return fmt.Errorf("failed to pull the result for delete resource group: %w", err)
-	}
-
 	log.Printf("resource group \"%s\" deleted successfully", d.ResourceGroupName)
 	return nil
 }
@@ -47,6 +41,6 @@ func (d *DeleteResourceGroup) Prevalidate() error {
 	return nil
 }
 
-func (d *DeleteResourceGroup) Postvalidate() error {
+func (d *DeleteResourceGroup) Stop() error {
 	return nil
 }

test/e2e/framework/azure/enable-ama.go

@@ -112,6 +112,6 @@ func (c *CreateAzureMonitor) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateAzureMonitor) Postvalidate() error {
+func (c *CreateAzureMonitor) Stop() error {
 	return nil
 }

test/e2e/framework/azure/get-kubeconfig.go

@@ -48,6 +48,6 @@ func (c *GetAKSKubeConfig) Prevalidate() error {
 	return nil
 }
 
-func (c *GetAKSKubeConfig) Postvalidate() error {
+func (c *GetAKSKubeConfig) Stop() error {
 	return nil
 }

test/e2e/framework/kubernetes/create-agnhost-statefulset.go

@@ -19,11 +19,6 @@ var ErrLabelMissingFromPod = fmt.Errorf("label missing from pod")
 const (
 	AgnhostHTTPPort = 80
 	AgnhostReplicas = 1
-
-	defaultTimeoutSeconds    = 300
-	defaultRetryDelay        = 5 * time.Second
-	defaultRetryAttempts     = 60
-	defaultHTTPClientTimeout = 2 * time.Second
 )
 
 type CreateAgnhostStatefulSet struct {
@@ -71,7 +66,7 @@ func (c *CreateAgnhostStatefulSet) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateAgnhostStatefulSet) Postvalidate() error {
+func (c *CreateAgnhostStatefulSet) Stop() error {
 	return nil
 }
 

test/e2e/framework/kubernetes/create-kapinger-deployment.go

@@ -70,7 +70,7 @@ func (c *CreateKapingerDeployment) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateKapingerDeployment) Postvalidate() error {
+func (c *CreateKapingerDeployment) Stop() error {
 	return nil
 }
 

test/e2e/framework/kubernetes/create-network-policy.go

@@ -72,7 +72,7 @@ func (c *CreateDenyAllNetworkPolicy) Prevalidate() error {
 	return nil
 }
 
-func (c *CreateDenyAllNetworkPolicy) Postvalidate() error {
+func (c *CreateDenyAllNetworkPolicy) Stop() error {
 	return nil
 }
 
@@ -108,7 +108,3 @@ func (d *DeleteDenyAllNetworkPolicy) Run() error {
 func (d *DeleteDenyAllNetworkPolicy) Prevalidate() error {
 	return nil
 }
-
-func (d *DeleteDenyAllNetworkPolicy) Postvalidate() error {
-	return nil
-}

test/e2e/framework/kubernetes/exec-pod.go

@@ -39,7 +39,7 @@ func (e *ExecInPod) Prevalidate() error {
 	return nil
 }
 
-func (e *ExecInPod) Postvalidate() error {
+func (e *ExecInPod) Stop() error {
 	return nil
 }
 

test/e2e/framework/kubernetes/port-forward.txt → test/e2e/framework/kubernetes/port-forward.go

@@ -39,8 +39,7 @@ type PortForward struct {
 	OptionalLabelAffinity string
 
 	// local properties
-	pf                *k8s.PortForwarder
-	portForwardHandle k8s.PortForwardStreamHandle
+	pf *k8s.PortForwarder
 }
 
 func (p *PortForward) Run() error {
@@ -61,61 +60,63 @@ func (p *PortForward) Run() error {
 		return fmt.Errorf("could not create clientset: %w", err)
 	}
 
-	p.pf, err = k8s.NewPortForwarder(config)
-	if err != nil {
-		return fmt.Errorf("could not create port forwarder: %w", err)
-	}
-
 	// if we have an optional label affinity, find a pod with that label, on the same node as a pod with the label selector
 	targetPodName := ""
 	if p.OptionalLabelAffinity != "" {
 		// get all pods with label
+		log.Printf("attempting to find pod with label \"%s\", on a node with a pod with label \"%s\"\n", p.LabelSelector, p.OptionalLabelAffinity)
 		targetPodName, err = p.findPodsWithAffinity(pctx, clientset)
 		if err != nil {
 			return fmt.Errorf("could not find pod with affinity: %w", err)
 		}
 	}
 
 	portForwardFn := func() error {
-		log.Printf("attempting port forward to a pod with label \"%s\", in namespace \"%s\"...\n", p.LabelSelector, p.Namespace)
-
-		var handle k8s.PortForwardStreamHandle
 
 		// if we have a pod name (likely from affinity above), use it, otherwise use label selector
+		opts := k8s.PortForwardingOpts{
+			Namespace: p.Namespace,
+			PodName:   targetPodName,
+			LocalPort: lport,
+			DestPort:  rport,
+		}
+
 		if targetPodName != "" {
-			handle, err = p.pf.ForwardWithPodName(pctx, p.Namespace, targetPodName, lport, rport)
-			if err != nil {
-				return fmt.Errorf("could not start port forward: %w", err)
-			}
-		} else {
-			handle, err = p.pf.ForwardWithLabelSelector(pctx, p.Namespace, p.LabelSelector, lport, rport)
-			if err != nil {
-				return fmt.Errorf("could not start port forward: %w", err)
-			}
+			opts.PodName = targetPodName
+		}
+
+		log.Printf("attempting port forward to pod name \"%s\" with label \"%s\", in namespace \"%s\"...\n", targetPodName, p.LabelSelector, p.Namespace)
+
+		p.pf, err = k8s.NewPortForwarder(config, &logger{}, opts)
+		if err != nil {
+			return fmt.Errorf("could not create port forwarder: %w", err)
+		}
+		err = p.pf.Forward(pctx)
+		if err != nil {
+			return fmt.Errorf("could not start port forward: %w", err)
 		}
 
 		// verify port forward succeeded
 		client := http.Client{
 			Timeout: defaultHTTPClientTimeout,
 		}
-		resp, err := client.Get(handle.URL()) //nolint
+		resp, err := client.Get(p.pf.Address()) //nolint
 		if err != nil {
-			log.Printf("port forward validation HTTP request to %s failed: %v\n", handle.URL(), err)
-			handle.Stop()
-			return fmt.Errorf("port forward validation HTTP request to %s failed: %w", handle.URL(), err)
+			log.Printf("port forward validation HTTP request to %s failed: %v\n", p.pf.Address(), err)
+			p.pf.Stop()
+			return fmt.Errorf("port forward validation HTTP request to %s failed: %w", p.pf.Address(), err)
 		}
 		defer resp.Body.Close()
 
-		log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", handle.URL(), resp.Status)
-		p.portForwardHandle = handle
+		log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", p.pf.Address(), resp.Status)
 
 		return nil
 	}
 
 	if err = defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil {
 		return fmt.Errorf("could not start port forward within %ds: %w", defaultTimeoutSeconds, err)
 	}
-	log.Printf("successfully port forwarded to \"%s\"\n", p.portForwardHandle.URL())
+	log.Printf("successfully port forwarded to \"%s\"\n", p.pf.Address())
 	return nil
 }
 
@@ -157,7 +158,13 @@ func (p *PortForward) Prevalidate() error {
 	return nil
 }
 
-func (p *PortForward) Postvalidate() error {
-	p.portForwardHandle.Stop()
+func (p *PortForward) Stop() error {
+	p.pf.Stop()
 	return nil
 }
+
+type logger struct{}
+
+func (l *logger) Logf(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}