From 3c43d7e8242e520c53df124194f5055a68b8a7a4 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Wed, 11 Oct 2023 09:51:05 -0700
Subject: [PATCH 01/27] [test] add hubble system test to CI (#2265)

* cilium configmap

* update hubble configs and add metrics test

* update pipeline yaml

* separate cilium+hubble config
---
 .gitignore                                    |   3 +
 .../cilium/cilium-e2e-step-template.yaml      |   8 +
 hack/toolbox/server/Dockerfile.heavy          |   6 +-
 test/integration/hubble/hubble_test.go        | 163 ++++++++++++++++++
 .../cilium/cilium-config-hubble.yaml          |  98 +++++++++++
 .../cilium/hubble/hubble-peer-svc.yaml        |  18 ++
 6 files changed, 293 insertions(+), 3 deletions(-)
 create mode 100644 test/integration/hubble/hubble_test.go
 create mode 100644 test/integration/manifests/cilium/cilium-config-hubble.yaml
 create mode 100644 test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml

diff --git a/.gitignore b/.gitignore
index 6ecb9304d9..e55c1f21fd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,6 @@ go.work*
 
 # scale-test
 test/scale/generated/*
+
+# test env file
+*.env
diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
index 914129241c..02c7a0f6c3 100644
--- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
@@ -41,6 +41,7 @@ steps:
         echo "deploy Cilium ConfigMap"
         kubectl apply -f cilium/configmap.yaml
         kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
+        kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
@@ -177,6 +178,13 @@ steps:
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
+  - script: |
+      echo "verify hubble metrics endpoint is usable"
+      go test ./test/integration/hubble/ -count=1 -v
+    retryCountOnTaskFailure: 3
+    name: "HubbleConnectivityTests"
+    displayName: "Run Hubble Connectivity Tests"
+
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
       echo $ARTIFACT_DIR
diff --git a/hack/toolbox/server/Dockerfile.heavy b/hack/toolbox/server/Dockerfile.heavy
index ee9aea25d2..fbcee1c15d 100644
--- a/hack/toolbox/server/Dockerfile.heavy
+++ b/hack/toolbox/server/Dockerfile.heavy
@@ -3,7 +3,7 @@ ADD ./ /
 WORKDIR /
 RUN CGO_ENABLED=0 GOOS=linux go build -o server .
 
-FROM mcr.microsoft.com/oss/mirror/docker.io/library/ubuntu:20.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
 RUN apt-get update
 RUN apt-get install -y \
     axel \
@@ -21,14 +21,14 @@ RUN apt-get install -y \
     net-tools \
     netcat \
     nmap \
-    python \
     python3 \
     ssh \
     sudo \
     tcpdump \
     traceroute \
+    unzip \
     vim \
-    wget
+    wget 
 
 RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
 RUN curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256"
diff --git a/test/integration/hubble/hubble_test.go b/test/integration/hubble/hubble_test.go
new file mode 100644
index 0000000000..1f7a57a2f2
--- /dev/null
+++ b/test/integration/hubble/hubble_test.go
@@ -0,0 +1,163 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	k8s "github.com/Azure/azure-container-networking/test/integration"
+	"github.com/Azure/azure-container-networking/test/internal/retry"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+)
+
+const (
+	retryAttempts = 10
+	retryDelay    = 5 * time.Second
+	promAddress   = "http://localhost:9965/metrics"
+	labelSelector = "k8s-app=cilium"
+	namespace     = "kube-system"
+)
+
+var (
+	defaultRetrier  = retry.Retrier{Attempts: retryAttempts, Delay: retryDelay}
+	requiredMetrics = []string{
+		"hubble_flows_processed_total",
+		"hubble_tcp_flags_total",
+	}
+)
+
+func TestEndpoints(t *testing.T) {
+	var kubeconfigPath string
+	flag.StringVar(&kubeconfigPath, "kubeconfig", getDefaultKubeconfigPath(), "Path to the kubeconfig file")
+	flag.Parse()
+
+	config, err := getClientConfig(kubeconfigPath)
+	if err != nil {
+		fmt.Printf("Error creating Kubernetes client config: %v\n", err)
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+	clusterCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+	pingCheckFn := func() error {
+		var pf *k8s.PortForwarder
+		pf, err = k8s.NewPortForwarder(config, t, k8s.PortForwardingOpts{
+			Namespace:     namespace,
+			LabelSelector: labelSelector,
+			LocalPort:     9965,
+			DestPort:      9965,
+		})
+		if err != nil {
+			t.Error(err)
+		}
+		pctx := context.Background()
+
+		portForwardCtx, cancel := context.WithTimeout(pctx, (retryAttempts+1)*retryDelay)
+		defer cancel()
+
+		portForwardFn := func() error {
+			t.Logf("attempting port forward to a pod with label %s, in namespace %s...", labelSelector, namespace)
+			if err := pf.Forward(portForwardCtx); err != nil {
+				return fmt.Errorf("could not start port forward: %w", err)
+			}
+			return nil
+		}
+
+		if err := defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil {
+			t.Fatalf("could not start port forward within %d: %v", (retryAttempts+1)*retryDelay, err)
+		}
+		defer pf.Stop()
+
+		// scrape the hubble metrics
+		metrics, err := getPrometheusMetrics(promAddress)
+		if err != nil {
+			return fmt.Errorf("scraping %s, failed with error: %w", promAddress, err)
+		}
+
+		// verify that the response contains the required metrics
+		for _, reqMetric := range requiredMetrics {
+			if val, exists := metrics[reqMetric]; !exists {
+				return fmt.Errorf("scraping %s, did not find metric %s", val, promAddress) //nolint:goerr113,gocritic
+			}
+		}
+		t.Logf("all metrics validated: %+v", requiredMetrics)
+		return nil
+	}
+
+	if err := defaultRetrier.Do(clusterCtx, pingCheckFn); err != nil {
+		t.Fatalf("metrics check failed with error: %v", err)
+	}
+}
+
+func getPrometheusMetrics(url string) (map[string]struct{}, error) {
+	client := http.Client{}
+	resp, err := client.Get(url) //nolint
+	if err != nil {
+		return nil, fmt.Errorf("HTTP request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP request failed with status: %v", resp.Status) //nolint:goerr113,gocritic
+	}
+
+	metricsData, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading HTTP response body failed: %w", err)
+	}
+
+	metrics := parseMetrics(string(metricsData))
+	return metrics, nil
+}
+
+func parseMetrics(metricsData string) map[string]struct{} {
+	// Create a map to store the strings before the first '{'.
+	metrics := make(map[string]struct{})
+
+	// sample metrics
+	// hubble_tcp_flags_total{destination="",family="IPv4",flag="RST",source="kube-system/metrics-server"} 980
+	// hubble_tcp_flags_total{destination="",family="IPv4",flag="SYN",source="kube-system/ama-metrics"} 1777
+	// we only want the metric name for the time being
+	// label order/parseing can happen later
+	lines := strings.Split(metricsData, "\n")
+	// Iterate through each line.
+	for _, line := range lines {
+		// Find the index of the first '{' character.
+		index := strings.Index(line, "{")
+		if index >= 0 {
+			// Extract the string before the first '{'.
+			str := strings.TrimSpace(line[:index])
+			// Store the string in the map.
+			metrics[str] = struct{}{}
+		}
+	}
+
+	return metrics
+}
+
+func getDefaultKubeconfigPath() string {
+	home := homedir.HomeDir()
+	return filepath.Join(home, ".kube", "config")
+}
+
+func getClientConfig(kubeconfigPath string) (*rest.Config, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		// If running outside a Kubernetes cluster, use the kubeconfig file.
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+		if err != nil {
+			return nil, fmt.Errorf("error creating Kubernetes client config: %w", err)
+		}
+	}
+	return config, nil
+}
diff --git a/test/integration/manifests/cilium/cilium-config-hubble.yaml b/test/integration/manifests/cilium/cilium-config-hubble.yaml
new file mode 100644
index 0000000000..42ea460c77
--- /dev/null
+++ b/test/integration/manifests/cilium/cilium-config-hubble.yaml
@@ -0,0 +1,98 @@
+apiVersion: v1
+data:
+  agent-not-ready-taint-key: node.cilium.io/agent-not-ready
+  arping-refresh-period: 30s
+  auto-direct-node-routes: "false"
+  bpf-lb-external-clusterip: "false"
+  bpf-lb-map-max: "65536"
+  bpf-lb-mode: snat
+  bpf-map-dynamic-size-ratio: "0.0025"
+  bpf-policy-map-max: "16384"
+  bpf-root: /sys/fs/bpf
+  cgroup-root: /run/cilium/cgroupv2
+  cilium-endpoint-gc-interval: 5m0s
+  cluster-id: "0"
+  cluster-name: default
+  debug: "false"
+  disable-cnp-status-updates: "true"
+  disable-endpoint-crd: "false"
+  enable-auto-protect-node-port-range: "true"
+  enable-bgp-control-plane: "false"
+  enable-bpf-clock-probe: "true"
+  enable-endpoint-health-checking: "false"
+  enable-endpoint-routes: "true"
+  enable-health-check-nodeport: "true"
+  enable-health-checking: "true"
+  enable-host-legacy-routing: "true"
+  enable-hubble: "true"
+  enable-ipv4: "true"
+  enable-ipv4-masquerade: "false"
+  enable-ipv6: "false"
+  enable-ipv6-masquerade: "false"
+  enable-k8s-terminating-endpoint: "true"
+  enable-l2-neigh-discovery: "true"
+  enable-l7-proxy: "false"
+  enable-local-node-route: "false"
+  enable-local-redirect-policy: "false"
+  enable-metrics: "true"
+  enable-policy: default
+  enable-remote-node-identity: "true"
+  enable-session-affinity: "true"
+  enable-svc-source-range-check: "true"
+  enable-vtep: "false"
+  enable-well-known-identities: "false"
+  enable-xt-socket-fallback: "true"
+  hubble-metrics: flow:sourceContext=pod-short;destinationContext=pod-short
+    tcp:sourceContext=pod-short;destinationContext=pod-short
+    dns:flow:sourceContext=pod-short;destinationContext=pod-short
+  hubble-metrics-server: :9965
+  hubble-disable-tls: "false"
+  hubble-listen-address: ""
+  hubble-socket-path: /dev/null
+  hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt
+  hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt
+  hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key
+  identity-allocation-mode: crd
+  install-iptables-rules: "true"
+  install-no-conntrack-iptables-rules: "false"
+  ipam: delegated-plugin
+  kube-proxy-replacement: strict
+  kube-proxy-replacement-healthz-bind-address: "0.0.0.0:10256"
+  local-router-ipv4: 169.254.23.0
+  metrics: +cilium_bpf_map_pressure
+  monitor-aggregation: medium
+  monitor-aggregation-flags: all
+  monitor-aggregation-interval: 5s
+  node-port-bind-protection: "true"
+  nodes-gc-interval: 5m0s
+  operator-api-serve-addr: 127.0.0.1:9234
+  operator-prometheus-serve-addr: :9963
+  preallocate-bpf-maps: "false"
+  procfs: /host/proc
+  prometheus-serve-addr: :9962
+  remove-cilium-node-taints: "true"
+  set-cilium-is-up-condition: "true"
+  sidecar-istio-proxy-image: cilium/istio_proxy
+  synchronize-k8s-nodes: "true"
+  tofqdns-dns-reject-response-code: refused
+  tofqdns-enable-dns-compression: "true"
+  tofqdns-endpoint-max-ip-per-hostname: "50"
+  tofqdns-idle-connection-grace-period: 0s
+  tofqdns-max-deferred-connection-deletes: "10000"
+  tofqdns-min-ttl: "3600"
+  tofqdns-proxy-response-max-delay: 100ms
+  tunnel: disabled
+  unmanaged-pod-watcher-interval: "15"
+  vtep-cidr: ""
+  vtep-endpoint: ""
+  vtep-mac: ""
+  vtep-mask: ""
+kind: ConfigMap
+metadata:
+  annotations:
+    meta.helm.sh/release-name: cilium
+    meta.helm.sh/release-namespace: kube-system
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: cilium-config
+  namespace: kube-system
diff --git a/test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml b/test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
new file mode 100644
index 0000000000..6ba733885c
--- /dev/null
+++ b/test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: cilium
+  name: hubble-peer
+  namespace: kube-system
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: peer-service
+      port: 443
+      protocol: TCP
+      targetPort: 4244
+  selector:
+    k8s-app: cilium
+  sessionAffinity: None
+  type: ClusterIP

From 8d1816b8c107545134e1b36eb01d5cad896f7a12 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Wed, 11 Oct 2023 13:58:20 -0700
Subject: [PATCH 02/27] ci: move hubble connectivity test to nightly pipeline

---
 .../cni/cilium/cilium-overlay-load-test-template.yaml     | 7 +++++++
 .../singletenancy/cilium/cilium-e2e-step-template.yaml    | 8 --------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
index e8a3498c0a..2198c81361 100644
--- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
+++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
@@ -60,6 +60,7 @@ stages:
                 echo "deploy Cilium ConfigMap"
                 kubectl apply -f cilium/configmap.yaml
                 kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
+                kubectl apply -f test/integration/manifests/cililum/hubble/hubble-peer-svc.yaml
                 echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster"
                 # Passes Cilium image to daemonset and deployment
                 envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
@@ -142,6 +143,12 @@ stages:
             retryCountOnTaskFailure: 6
             name: "CiliumConnectivityTests"
             displayName: "Run Cilium Connectivity Tests"
+          - script: |
+              echo "verify Hubble metrics endpoint is usable"
+              go test ./test/integration/hubble/ -count=1 -v
+            retryCountOnTaskFailure: 6
+            name: "HubbleConnectivityTests"
+            displayName: "Run Hubble Connectivity Tests"
       - job: failedE2ELogs
         displayName: "Failure Logs"
         dependsOn:
diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
index 02c7a0f6c3..914129241c 100644
--- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
@@ -41,7 +41,6 @@ steps:
         echo "deploy Cilium ConfigMap"
         kubectl apply -f cilium/configmap.yaml
         kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
-        kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
@@ -178,13 +177,6 @@ steps:
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
-  - script: |
-      echo "verify hubble metrics endpoint is usable"
-      go test ./test/integration/hubble/ -count=1 -v
-    retryCountOnTaskFailure: 3
-    name: "HubbleConnectivityTests"
-    displayName: "Run Hubble Connectivity Tests"
-
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
       echo $ARTIFACT_DIR

From 2b11ae9a178f759806795367e667ddc6ea42ffdf Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Wed, 11 Oct 2023 15:18:54 -0700
Subject: [PATCH 03/27] fix: move to correct file

---
 .../cni/cilium/cilium-overlay-load-test-template.yaml     | 7 -------
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml  | 8 ++++++++
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
index 2198c81361..e8a3498c0a 100644
--- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
+++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
@@ -60,7 +60,6 @@ stages:
                 echo "deploy Cilium ConfigMap"
                 kubectl apply -f cilium/configmap.yaml
                 kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
-                kubectl apply -f test/integration/manifests/cililum/hubble/hubble-peer-svc.yaml
                 echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster"
                 # Passes Cilium image to daemonset and deployment
                 envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
@@ -143,12 +142,6 @@ stages:
             retryCountOnTaskFailure: 6
             name: "CiliumConnectivityTests"
             displayName: "Run Cilium Connectivity Tests"
-          - script: |
-              echo "verify Hubble metrics endpoint is usable"
-              go test ./test/integration/hubble/ -count=1 -v
-            retryCountOnTaskFailure: 6
-            name: "HubbleConnectivityTests"
-            displayName: "Run Hubble Connectivity Tests"
       - job: failedE2ELogs
         displayName: "Failure Logs"
         dependsOn:
diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 8731f6ed5f..712509e3cb 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -42,6 +42,7 @@ steps:
         echo "deploy Cilium ConfigMap"
         kubectl apply -f cilium/configmap.yaml
         kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-config.yaml
+        kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         # Passes Cilium image to daemonset and deployment
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
@@ -218,6 +219,13 @@ steps:
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
+  - script: |
+            echo "verify Hubble metrics endpoint is usable"
+            go test ./test/integration/hubble/ -count=1 -v
+          retryCountOnTaskFailure: 3
+          name: "HubbleConnectivityTests"
+          displayName: "Run Hubble Connectivity Tests"
+
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
       echo $ARTIFACT_DIR

From 07073fcb460e91ef9a4c9438ec04d9e227764518 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Wed, 11 Oct 2023 15:29:42 -0700
Subject: [PATCH 04/27] style: indentation change

---
 .../cilium-overlay-e2e-step-template.yaml              | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 712509e3cb..f415f4a5aa 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -220,11 +220,11 @@ steps:
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
   - script: |
-            echo "verify Hubble metrics endpoint is usable"
-            go test ./test/integration/hubble/ -count=1 -v
-          retryCountOnTaskFailure: 3
-          name: "HubbleConnectivityTests"
-          displayName: "Run Hubble Connectivity Tests"
+      echo "verify Hubble metrics endpoint is usable"
+      go test ./test/integration/hubble/ -count=1 -v
+    retryCountOnTaskFailure: 3
+    name: "HubbleConnectivityTests"
+    displayName: "Run Hubble Connectivity Tests"
 
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/

From cc5cffa17ea549e5ae1d96e7a886a0ebccc63ee4 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Thu, 12 Oct 2023 14:29:36 -0700
Subject: [PATCH 05/27] ci: update configmap to enable Hubble

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index f415f4a5aa..344fb03928 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -220,6 +220,11 @@ steps:
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
   - script: |
+      echo "update Cilium configmap to enable Hubble" 
+      kubectl get configmaps -A
+      kubectl replace -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+      kubectl rollout restart ds cilium
+      kubectl get configmaps -A
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/integration/hubble/ -count=1 -v
     retryCountOnTaskFailure: 3

From e684c32c26c61433d581482436f44fab77f894d1 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Thu, 12 Oct 2023 16:01:46 -0700
Subject: [PATCH 06/27] fix: move hubble test

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml        | 2 +-
 test/{integration => }/hubble/hubble_test.go                    | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test/{integration => }/hubble/hubble_test.go (100%)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 344fb03928..119cc673d8 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -226,7 +226,7 @@ steps:
       kubectl rollout restart ds cilium
       kubectl get configmaps -A
       echo "verify Hubble metrics endpoint is usable"
-      go test ./test/integration/hubble/ -count=1 -v
+      go test ./test/hubble/ -count=1 -v
     retryCountOnTaskFailure: 3
     name: "HubbleConnectivityTests"
     displayName: "Run Hubble Connectivity Tests"
diff --git a/test/integration/hubble/hubble_test.go b/test/hubble/hubble_test.go
similarity index 100%
rename from test/integration/hubble/hubble_test.go
rename to test/hubble/hubble_test.go

From f0a6bd73a92942b58c5d5d61d7ef142240bf9664 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Thu, 12 Oct 2023 16:56:32 -0700
Subject: [PATCH 07/27] fix: move connectivity test before delete

---
 .../cilium-overlay-e2e-step-template.yaml     | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 119cc673d8..ff95a18a50 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -153,6 +153,20 @@ steps:
     name: "ciliumConnectivityTests"
     displayName: "Run Cilium Connectivity Tests"
 
+  - script: |
+      echo "update Cilium configmap to enable Hubble" 
+      kubectl get pods -A
+      kubectl get daemonset -A
+      kubectl get configmaps -A
+      kubectl replace -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+      kubectl rollout restart ds cilium
+      kubectl get configmaps -A
+      echo "verify Hubble metrics endpoint is usable"
+      go test ./test/hubble/ -count=1 -v
+    retryCountOnTaskFailure: 3
+    name: "HubbleConnectivityTests"
+    displayName: "Run Hubble Connectivity Tests"
+
   - script: |
       echo "validate pod IP assignment and check systemd-networkd restart"
       kubectl get pod -owide -A
@@ -219,18 +233,6 @@ steps:
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
-  - script: |
-      echo "update Cilium configmap to enable Hubble" 
-      kubectl get configmaps -A
-      kubectl replace -f test/integration/manifests/cilium/cilium-config-hubble.yaml
-      kubectl rollout restart ds cilium
-      kubectl get configmaps -A
-      echo "verify Hubble metrics endpoint is usable"
-      go test ./test/hubble/ -count=1 -v
-    retryCountOnTaskFailure: 3
-    name: "HubbleConnectivityTests"
-    displayName: "Run Hubble Connectivity Tests"
-
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
       echo $ARTIFACT_DIR

From a7d4b1b165a16b2fc82911b74fbc86030e48707e Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Thu, 12 Oct 2023 19:39:09 -0700
Subject: [PATCH 08/27] fix: add daemonset namespace

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index ff95a18a50..417ef39019 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -159,7 +159,7 @@ steps:
       kubectl get daemonset -A
       kubectl get configmaps -A
       kubectl replace -f test/integration/manifests/cilium/cilium-config-hubble.yaml
-      kubectl rollout restart ds cilium
+      kubectl rollout restart ds cilium -n kube-system
       kubectl get configmaps -A
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v

From b7766e559660434557d8944a827fca940a85db9b Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 09:47:46 -0700
Subject: [PATCH 09/27] fix: update command for configmap replace

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 417ef39019..1dfe383632 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -158,9 +158,11 @@ steps:
       kubectl get pods -A
       kubectl get daemonset -A
       kubectl get configmaps -A
-      kubectl replace -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+      kubectl describe cm cilium-config -n kube-system
+      kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl replace -f -
       kubectl rollout restart ds cilium -n kube-system
       kubectl get configmaps -A
+      kubectl describe cm cilium-config -n kube-system
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v
     retryCountOnTaskFailure: 3

From 838340f0d4558698bd97f1b721406ffdb7414175 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 10:50:09 -0700
Subject: [PATCH 10/27] test no restart after replace

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml         | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 1dfe383632..cdd6a38118 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -160,7 +160,6 @@ steps:
       kubectl get configmaps -A
       kubectl describe cm cilium-config -n kube-system
       kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl replace -f -
-      kubectl rollout restart ds cilium -n kube-system
       kubectl get configmaps -A
       kubectl describe cm cilium-config -n kube-system
       echo "verify Hubble metrics endpoint is usable"

From 0b5bb1957405e73ff751154d3cdd47860f9f8dd3 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 12:40:25 -0700
Subject: [PATCH 11/27] fix: try apply instead of replace

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index cdd6a38118..62ae65ad56 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -154,13 +154,15 @@ steps:
     displayName: "Run Cilium Connectivity Tests"
 
   - script: |
-      echo "update Cilium configmap to enable Hubble" 
+      echo "update Cilium configmap to enable Hubble"
       kubectl get pods -A
       kubectl get daemonset -A
       kubectl get configmaps -A
       kubectl describe cm cilium-config -n kube-system
-      kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl replace -f -
+      kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
       kubectl get configmaps -A
+      sleep 20s
+      kubectl get pods -A
       kubectl describe cm cilium-config -n kube-system
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v

From 050d4223c206b5e2f53e29d9a7bd2c6c1cfd7706 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 12:41:46 -0700
Subject: [PATCH 12/27] fix: add back restart ds

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml         | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 62ae65ad56..b1f4be585b 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -160,6 +160,7 @@ steps:
       kubectl get configmaps -A
       kubectl describe cm cilium-config -n kube-system
       kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
+      kubectl rollout restart ds cilium -n kube-system
       kubectl get configmaps -A
       sleep 20s
       kubectl get pods -A

From 481af23ee51c445a376983a1bacdcfa521d9f326 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 14:08:08 -0700
Subject: [PATCH 13/27] add longer timeout after ds restart

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index b1f4be585b..3dcb494bab 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -154,17 +154,18 @@ steps:
     displayName: "Run Cilium Connectivity Tests"
 
   - script: |
-      echo "update Cilium configmap to enable Hubble"
+      echo "Run HubbleConnectivityTests"
       kubectl get pods -A
       kubectl get daemonset -A
       kubectl get configmaps -A
       kubectl describe cm cilium-config -n kube-system
+      echo "update Cilium configmap to enable Hubble"
       kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
       kubectl rollout restart ds cilium -n kube-system
-      kubectl get configmaps -A
-      sleep 20s
-      kubectl get pods -A
       kubectl describe cm cilium-config -n kube-system
+      echo "wait < 2 minutes for pods to be ready after restart"
+      kubectl wait pod --all --for=condition=Ready --namespace=kube-system --timeout=120s
+      kubectl get pods -Aowide
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v
     retryCountOnTaskFailure: 3

From 8ebb5adf5f407590652129e60cb76baf61bc8278 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 14:53:36 -0700
Subject: [PATCH 14/27] adjust timeout setup

---
 .../cilium-overlay-e2e-step-template.yaml              | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 3dcb494bab..80c47149fb 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -163,8 +163,14 @@ steps:
       kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
       kubectl rollout restart ds cilium -n kube-system
       kubectl describe cm cilium-config -n kube-system
-      echo "wait < 2 minutes for pods to be ready after restart"
-      kubectl wait pod --all --for=condition=Ready --namespace=kube-system --timeout=120s
+      sleep 5s
+      kubectl get pods -Aowide
+      echo "wait 3 minutes for pods to be ready after restart"
+      sleep 60s
+      kubectl get pods -Aowide
+      sleep 60s
+      kubectl get pods -Aowide
+      sleep 60s
       kubectl get pods -Aowide
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v

From daa576dd6187976a39c46bc25244915bee76e32b Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Fri, 13 Oct 2023 16:04:52 -0700
Subject: [PATCH 15/27] extend timeout, add logging

---
 .../cilium-overlay-e2e-step-template.yaml       | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 80c47149fb..4b4a1daed8 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -165,16 +165,17 @@ steps:
       kubectl describe cm cilium-config -n kube-system
       sleep 5s
       kubectl get pods -Aowide
-      echo "wait 3 minutes for pods to be ready after restart"
-      sleep 60s
-      kubectl get pods -Aowide
-      sleep 60s
-      kubectl get pods -Aowide
-      sleep 60s
-      kubectl get pods -Aowide
+      echo "wait 10 minutes for pods to be ready after restart"
+      for i in {0..9}
+      do
+        echo "minute $i"
+        sleep 60s
+        kubectl get pods -Aowide
+      done
+      kubectl describe pods -n kube-system
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v
-    retryCountOnTaskFailure: 3
+    retryCountOnTaskFailure: 1
     name: "HubbleConnectivityTests"
     displayName: "Run Hubble Connectivity Tests"
 

From ac77e113090d7cd006e650946a1e6764fec10c90 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Mon, 16 Oct 2023 10:05:58 -0700
Subject: [PATCH 16/27] add logging, change cm command

---
 .../cilium-overlay-e2e-step-template.yaml           | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 4b4a1daed8..40f0d52045 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -155,14 +155,20 @@ steps:
 
   - script: |
       echo "Run HubbleConnectivityTests"
+      echo "DEBUG / get pods"
       kubectl get pods -A
+      echo "DEBUG / get daemonset"
       kubectl get daemonset -A
+      echo "DEBUG / get configmaps"
       kubectl get configmaps -A
-      kubectl describe cm cilium-config -n kube-system
+      echo "DEBUG / get cilium-config"
+      kubectl get configmap cilium-config -n kube-system --output yaml
       echo "update Cilium configmap to enable Hubble"
-      kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
+      # kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
+      kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
       kubectl rollout restart ds cilium -n kube-system
-      kubectl describe cm cilium-config -n kube-system
+      echo "DEBUG / get cilium-config"
+      kubectl get configmap cilium-config -n kube-system --output yaml
       sleep 5s
       kubectl get pods -Aowide
       echo "wait 10 minutes for pods to be ready after restart"
@@ -172,6 +178,7 @@ steps:
         sleep 60s
         kubectl get pods -Aowide
       done
+      echo "DEBUG / pod logs"
       kubectl describe pods -n kube-system
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v

From 0c7953778aba864bd87dc20c46657b939da09f51 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Mon, 16 Oct 2023 16:33:53 -0700
Subject: [PATCH 17/27] update hubble configmap

---
 .../cilium-overlay-e2e-step-template.yaml     | 24 ++++---------------
 .../cilium/cilium-config-hubble.yaml          |  6 ++---
 2 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 40f0d52045..010aeabb5a 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -154,35 +154,21 @@ steps:
     displayName: "Run Cilium Connectivity Tests"
 
   - script: |
-      echo "Run HubbleConnectivityTests"
-      echo "DEBUG / get pods"
-      kubectl get pods -A
-      echo "DEBUG / get daemonset"
-      kubectl get daemonset -A
-      echo "DEBUG / get configmaps"
-      kubectl get configmaps -A
-      echo "DEBUG / get cilium-config"
-      kubectl get configmap cilium-config -n kube-system --output yaml
+      echo "Run Hubble Connectivity Tests"
       echo "update Cilium configmap to enable Hubble"
-      # kubectl create configmap cilium-config -n kube-system --from-file=test/integration/manifests/cilium/cilium-config-hubble.yaml -o yaml --dry-run=client | kubectl apply -f -
       kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
       kubectl rollout restart ds cilium -n kube-system
-      echo "DEBUG / get cilium-config"
-      kubectl get configmap cilium-config -n kube-system --output yaml
       sleep 5s
-      kubectl get pods -Aowide
-      echo "wait 10 minutes for pods to be ready after restart"
-      for i in {0..9}
+      echo "wait 3 minutes for pods to be ready after restart"
+      for i in {0..2}
       do
         echo "minute $i"
         sleep 60s
-        kubectl get pods -Aowide
       done
-      echo "DEBUG / pod logs"
-      kubectl describe pods -n kube-system
+      kubectl get pods -Aowide
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v
-    retryCountOnTaskFailure: 1
+    retryCountOnTaskFailure: 3
     name: "HubbleConnectivityTests"
     displayName: "Run Hubble Connectivity Tests"
 
diff --git a/test/integration/manifests/cilium/cilium-config-hubble.yaml b/test/integration/manifests/cilium/cilium-config-hubble.yaml
index 42ea460c77..c137aa23b4 100644
--- a/test/integration/manifests/cilium/cilium-config-hubble.yaml
+++ b/test/integration/manifests/cilium/cilium-config-hubble.yaml
@@ -42,9 +42,9 @@ data:
   enable-vtep: "false"
   enable-well-known-identities: "false"
   enable-xt-socket-fallback: "true"
-  hubble-metrics: flow:sourceContext=pod-short;destinationContext=pod-short
-    tcp:sourceContext=pod-short;destinationContext=pod-short
-    dns:flow:sourceContext=pod-short;destinationContext=pod-short
+  hubble-metrics: flow:sourceContext=workload-name;destinationContext=workload-name
+    tcp:sourceContext=workload-name;destinationContext=workload-name
+    dns:flow:sourceContext=workload-name;destinationContext=workload-name
   hubble-metrics-server: :9965
   hubble-disable-tls: "false"
   hubble-listen-address: ""

From 281de476e560a3b80bd50d0da3c4f10be2d59679 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Tue, 17 Oct 2023 09:23:27 -0700
Subject: [PATCH 18/27] clean up sleep statements

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml   | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 010aeabb5a..cb53db232e 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -158,13 +158,8 @@ steps:
       echo "update Cilium configmap to enable Hubble"
       kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
       kubectl rollout restart ds cilium -n kube-system
-      sleep 5s
       echo "wait 3 minutes for pods to be ready after restart"
-      for i in {0..2}
-      do
-        echo "minute $i"
-        sleep 60s
-      done
+      sleep 180s
       kubectl get pods -Aowide
       echo "verify Hubble metrics endpoint is usable"
       go test ./test/hubble/ -count=1 -v

From 4350e5d1e3bd00b91722bcea42b52a8b3453b46f Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Tue, 17 Oct 2023 11:48:29 -0700
Subject: [PATCH 19/27] remove hubble connectivity test from PR pipeline

---
 .../cilium-overlay-e2e-step-template.yaml     | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index cb53db232e..2f13a055aa 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -154,15 +154,19 @@ steps:
     displayName: "Run Cilium Connectivity Tests"
 
   - script: |
-      echo "Run Hubble Connectivity Tests"
-      echo "update Cilium configmap to enable Hubble"
-      kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
-      kubectl rollout restart ds cilium -n kube-system
-      echo "wait 3 minutes for pods to be ready after restart"
-      sleep 180s
-      kubectl get pods -Aowide
-      echo "verify Hubble metrics endpoint is usable"
-      go test ./test/hubble/ -count=1 -v
+      if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then
+        echo "Run Hubble Connectivity Tests"
+        echo "update Cilium configmap to enable Hubble"
+        kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+        kubectl rollout restart ds cilium -n kube-system
+        echo "wait 3 minutes for pods to be ready after restart"
+        sleep 180s
+        kubectl get pods -Aowide
+        echo "verify Hubble metrics endpoint is usable"
+        go test ./test/hubble/ -count=1 -v
+      else
+        echo "skip Hubble Connectivity Tests for PR pipeline"
+      fi
     retryCountOnTaskFailure: 3
     name: "HubbleConnectivityTests"
     displayName: "Run Hubble Connectivity Tests"

From 9770a9f19d57e0931e8620b5852e63ac1b984bc9 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Tue, 17 Oct 2023 13:31:53 -0700
Subject: [PATCH 20/27] use kubernetes utils

---
 test/hubble/hubble_test.go | 27 ++-------------------------
 1 file changed, 2 insertions(+), 25 deletions(-)

diff --git a/test/hubble/hubble_test.go b/test/hubble/hubble_test.go
index 1f7a57a2f2..3a38b88271 100644
--- a/test/hubble/hubble_test.go
+++ b/test/hubble/hubble_test.go
@@ -2,20 +2,17 @@ package main
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"io"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	k8s "github.com/Azure/azure-container-networking/test/integration"
+	"github.com/Azure/azure-container-networking/test/internal/kubernetes"
 	"github.com/Azure/azure-container-networking/test/internal/retry"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/homedir"
 )
 
@@ -36,15 +33,7 @@ var (
 )
 
 func TestEndpoints(t *testing.T) {
-	var kubeconfigPath string
-	flag.StringVar(&kubeconfigPath, "kubeconfig", getDefaultKubeconfigPath(), "Path to the kubeconfig file")
-	flag.Parse()
-
-	config, err := getClientConfig(kubeconfigPath)
-	if err != nil {
-		fmt.Printf("Error creating Kubernetes client config: %v\n", err)
-		os.Exit(1)
-	}
+	config := kubernetes.MustGetRestConfig()
 
 	ctx := context.Background()
 	clusterCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
@@ -149,15 +138,3 @@ func getDefaultKubeconfigPath() string {
 	home := homedir.HomeDir()
 	return filepath.Join(home, ".kube", "config")
 }
-
-func getClientConfig(kubeconfigPath string) (*rest.Config, error) {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		// If running outside a Kubernetes cluster, use the kubeconfig file.
-		config, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath)
-		if err != nil {
-			return nil, fmt.Errorf("error creating Kubernetes client config: %w", err)
-		}
-	}
-	return config, nil
-}

From 2850e5f9a47beaa29ced257d69ba1f278a646d77 Mon Sep 17 00:00:00 2001
From: jshr-w <144164353+jshr-w@users.noreply.github.com>
Date: Tue, 17 Oct 2023 13:33:56 -0700
Subject: [PATCH 21/27] fix style

Signed-off-by: jshr-w <144164353+jshr-w@users.noreply.github.com>
---
 hack/toolbox/server/Dockerfile.heavy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/toolbox/server/Dockerfile.heavy b/hack/toolbox/server/Dockerfile.heavy
index fbcee1c15d..6839f45794 100644
--- a/hack/toolbox/server/Dockerfile.heavy
+++ b/hack/toolbox/server/Dockerfile.heavy
@@ -28,7 +28,7 @@ RUN apt-get install -y \
     traceroute \
     unzip \
     vim \
-    wget 
+    wget
 
 RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
 RUN curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256"

From d3bca630876f1040ae28de41f57e35e14079b1d2 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Tue, 17 Oct 2023 14:33:25 -0700
Subject: [PATCH 22/27] update ds restart wait

---
 .../cilium-overlay-e2e-step-template.yaml          |  4 ++--
 test/hubble/hubble_test.go                         | 14 +++-----------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 2f13a055aa..2c8dbbf188 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -159,8 +159,8 @@ steps:
         echo "update Cilium configmap to enable Hubble"
         kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
         kubectl rollout restart ds cilium -n kube-system
-        echo "wait 3 minutes for pods to be ready after restart"
-        sleep 180s
+        echo "wait <3 minutes for pods to be ready after restart"
+        kubectl rollout status ds cilium -n kube-system --timeout=3m
         kubectl get pods -Aowide
         echo "verify Hubble metrics endpoint is usable"
         go test ./test/hubble/ -count=1 -v
diff --git a/test/hubble/hubble_test.go b/test/hubble/hubble_test.go
index 3a38b88271..656c25d91e 100644
--- a/test/hubble/hubble_test.go
+++ b/test/hubble/hubble_test.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -13,7 +12,6 @@ import (
 	k8s "github.com/Azure/azure-container-networking/test/integration"
 	"github.com/Azure/azure-container-networking/test/internal/kubernetes"
 	"github.com/Azure/azure-container-networking/test/internal/retry"
-	"k8s.io/client-go/util/homedir"
 )
 
 const (
@@ -34,13 +32,12 @@ var (
 
 func TestEndpoints(t *testing.T) {
 	config := kubernetes.MustGetRestConfig()
-
 	ctx := context.Background()
 	clusterCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 	defer cancel()
 	pingCheckFn := func() error {
 		var pf *k8s.PortForwarder
-		pf, err = k8s.NewPortForwarder(config, t, k8s.PortForwardingOpts{
+		pf, err := k8s.NewPortForwarder(config, t, k8s.PortForwardingOpts{
 			Namespace:     namespace,
 			LabelSelector: labelSelector,
 			LocalPort:     9965,
@@ -56,13 +53,13 @@ func TestEndpoints(t *testing.T) {
 
 		portForwardFn := func() error {
 			t.Logf("attempting port forward to a pod with label %s, in namespace %s...", labelSelector, namespace)
-			if err := pf.Forward(portForwardCtx); err != nil {
+			if err = pf.Forward(portForwardCtx); err != nil {
 				return fmt.Errorf("could not start port forward: %w", err)
 			}
 			return nil
 		}
 
-		if err := defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil {
+		if err = defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil {
 			t.Fatalf("could not start port forward within %d: %v", (retryAttempts+1)*retryDelay, err)
 		}
 		defer pf.Stop()
@@ -133,8 +130,3 @@ func parseMetrics(metricsData string) map[string]struct{} {
 
 	return metrics
 }
-
-func getDefaultKubeconfigPath() string {
-	home := homedir.HomeDir()
-	return filepath.Join(home, ".kube", "config")
-}

From 5397746f606d6d0501e976141f12a7540b4227bc Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Wed, 18 Oct 2023 09:32:28 -0700
Subject: [PATCH 23/27] enable Hubble on nightly, disable on PR

---
 .../cni/cilium/nightly-release-test.yml       |  1 +
 .../cilium-overlay-e2e-step-template.yaml     | 38 ++++++++++---------
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/.pipelines/cni/cilium/nightly-release-test.yml b/.pipelines/cni/cilium/nightly-release-test.yml
index ee43ece998..9cb7acc9ae 100644
--- a/.pipelines/cni/cilium/nightly-release-test.yml
+++ b/.pipelines/cni/cilium/nightly-release-test.yml
@@ -88,6 +88,7 @@ stages:
             name: "cilium_nightly"
             testDropgz: ""
             clusterName: ciliumnightly-$(commitID)
+            testHubble: true
       - job: logs
         displayName: "Failure Logs"
         dependsOn:
diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 2c8dbbf188..61d18659c4 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -2,6 +2,7 @@ parameters:
   name: ""
   testDropgz: ""
   clusterName: ""
+  testHubble: false
 
 steps:
   - bash: |
@@ -42,7 +43,6 @@ steps:
         echo "deploy Cilium ConfigMap"
         kubectl apply -f cilium/configmap.yaml
         kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-config.yaml
-        kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         # Passes Cilium image to daemonset and deployment
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
@@ -54,6 +54,18 @@ steps:
     name: "installCilium"
     displayName: "Install Cilium on AKS Overlay"
 
+  - ${{ if eq( parameters['testHubble'], true) }}:
+      - script: |
+          echo "enable Hubble metrics server"
+          kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
+          kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+          kubectl rollout restart ds cilium -n kube-system
+          echo "wait <3 minutes for pods to be ready after restart"
+          kubectl rollout status ds cilium -n kube-system --timeout=3m
+          kubectl get pods -Aowide
+        name: "installHubble"
+        displayName: "Install Hubble on AKS Overlay"
+
   - script: |
       echo "install cilium CLI"
       CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
@@ -153,23 +165,13 @@ steps:
     name: "ciliumConnectivityTests"
     displayName: "Run Cilium Connectivity Tests"
 
-  - script: |
-      if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then
-        echo "Run Hubble Connectivity Tests"
-        echo "update Cilium configmap to enable Hubble"
-        kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
-        kubectl rollout restart ds cilium -n kube-system
-        echo "wait <3 minutes for pods to be ready after restart"
-        kubectl rollout status ds cilium -n kube-system --timeout=3m
-        kubectl get pods -Aowide
-        echo "verify Hubble metrics endpoint is usable"
-        go test ./test/hubble/ -count=1 -v
-      else
-        echo "skip Hubble Connectivity Tests for PR pipeline"
-      fi
-    retryCountOnTaskFailure: 3
-    name: "HubbleConnectivityTests"
-    displayName: "Run Hubble Connectivity Tests"
+  - ${{ if eq( parameters['testHubble'], true) }}:
+      - script: |
+          echo "verify Hubble metrics endpoint is usable"
+          go test ./test/hubble/ -count=1 -v
+        retryCountOnTaskFailure: 3
+        name: "HubbleConnectivityTests"
+        displayName: "Run Hubble Connectivity Tests"
 
   - script: |
       echo "validate pod IP assignment and check systemd-networkd restart"

From 3b570cb4719dede3df6a549f8b15bbdf1f639ff6 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Wed, 18 Oct 2023 13:51:48 -0700
Subject: [PATCH 24/27] tag networkobservability test

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml      | 2 +-
 .../networkobservability}/hubble_test.go                      | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)
 rename test/{hubble => integration/networkobservability}/hubble_test.go (98%)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 61d18659c4..e3ab63e0d7 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -168,7 +168,7 @@ steps:
   - ${{ if eq( parameters['testHubble'], true) }}:
       - script: |
           echo "verify Hubble metrics endpoint is usable"
-          go test ./test/hubble/ -count=1 -v
+          go test ./test/hubble/ -count=1 -v -tags=networkobservability
         retryCountOnTaskFailure: 3
         name: "HubbleConnectivityTests"
         displayName: "Run Hubble Connectivity Tests"
diff --git a/test/hubble/hubble_test.go b/test/integration/networkobservability/hubble_test.go
similarity index 98%
rename from test/hubble/hubble_test.go
rename to test/integration/networkobservability/hubble_test.go
index 656c25d91e..4cae750e42 100644
--- a/test/hubble/hubble_test.go
+++ b/test/integration/networkobservability/hubble_test.go
@@ -1,4 +1,6 @@
-package main
+//go:build networkobservability
+
+package networkobservability
 
 import (
 	"context"

From 5c44984640e3bc0050bfe040b99ec0ace37836c8 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Wed, 18 Oct 2023 13:54:36 -0700
Subject: [PATCH 25/27] fix test call

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index e3ab63e0d7..d594b66264 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -168,7 +168,7 @@ steps:
   - ${{ if eq( parameters['testHubble'], true) }}:
       - script: |
           echo "verify Hubble metrics endpoint is usable"
-          go test ./test/hubble/ -count=1 -v -tags=networkobservability
+          go test ./test/integration/networkobservability -count=1 -v -tags=networkobservability
         retryCountOnTaskFailure: 3
         name: "HubbleConnectivityTests"
         displayName: "Run Hubble Connectivity Tests"

From 22b8975da5c6fe70947c486ea4585240eaad1d63 Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Thu, 19 Oct 2023 10:06:06 -0700
Subject: [PATCH 26/27] enable Hubble after Cilium is ready

---
 .../cilium-overlay-e2e-step-template.yaml     | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 3bd800d497..a4bcb43302 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -55,18 +55,6 @@ steps:
     name: "installCilium"
     displayName: "Install Cilium on AKS Overlay"
 
-  - ${{ if eq( parameters['testHubble'], true) }}:
-      - script: |
-          echo "enable Hubble metrics server"
-          kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
-          kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
-          kubectl rollout restart ds cilium -n kube-system
-          echo "wait <3 minutes for pods to be ready after restart"
-          kubectl rollout status ds cilium -n kube-system --timeout=3m
-          kubectl get pods -Aowide
-        name: "installHubble"
-        displayName: "Install Hubble on AKS Overlay"
-
   - script: |
       echo "install cilium CLI"
       CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
@@ -148,6 +136,18 @@ steps:
     name: "CiliumStatus"
     displayName: "Cilium Status"
 
+  - ${{ if eq( parameters['testHubble'], true) }}:
+      - script: |
+          echo "enable Hubble metrics server"
+          kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
+          kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+          kubectl rollout restart ds cilium -n kube-system
+          echo "wait <3 minutes for pods to be ready after restart"
+          kubectl rollout status ds cilium -n kube-system --timeout=3m
+          kubectl get pods -Aowide
+        name: "installHubble"
+        displayName: "Install Hubble on AKS Overlay"
+
   - script: |
       echo "Run Service Conformance E2E"
       export PATH=${PATH}:/usr/local/bin/gsutil

From d6d88b10318bbcfc0823add9e9ff72858fb0901e Mon Sep 17 00:00:00 2001
From: jshr-w <shjayaraman@microsoft.com>
Date: Thu, 19 Oct 2023 13:13:27 -0700
Subject: [PATCH 27/27] change location of hubble enable

---
 .../cilium-overlay-e2e-step-template.yaml     | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index a4bcb43302..88346909fb 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -136,18 +136,6 @@ steps:
     name: "CiliumStatus"
     displayName: "Cilium Status"
 
-  - ${{ if eq( parameters['testHubble'], true) }}:
-      - script: |
-          echo "enable Hubble metrics server"
-          kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
-          kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
-          kubectl rollout restart ds cilium -n kube-system
-          echo "wait <3 minutes for pods to be ready after restart"
-          kubectl rollout status ds cilium -n kube-system --timeout=3m
-          kubectl get pods -Aowide
-        name: "installHubble"
-        displayName: "Install Hubble on AKS Overlay"
-
   - script: |
       echo "Run Service Conformance E2E"
       export PATH=${PATH}:/usr/local/bin/gsutil
@@ -168,6 +156,13 @@ steps:
 
   - ${{ if eq( parameters['testHubble'], true) }}:
       - script: |
+          echo "enable Hubble metrics server"
+          kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
+          kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
+          kubectl rollout restart ds cilium -n kube-system
+          echo "wait <3 minutes for pods to be ready after restart"
+          kubectl rollout status ds cilium -n kube-system --timeout=3m
+          kubectl get pods -Aowide
           echo "verify Hubble metrics endpoint is usable"
           go test ./test/integration/networkobservability -count=1 -v -tags=networkobservability
         retryCountOnTaskFailure: 3