[test] add hubble system test to CI

.gitignore

@@ -35,3 +35,6 @@ go.work*
 
 # scale-test
 test/scale/generated/*
+
+# test env file
+*.env

.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml

@@ -41,6 +41,7 @@ steps:
         echo "deploy Cilium ConfigMap"
         kubectl apply -f cilium/configmap.yaml
         kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
+        kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
         envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
@@ -177,6 +178,13 @@ steps:
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
 
+  - script: |
+      echo "verify hubble metrics endpoint is usable"
+      go test ./test/integration/hubble/ -count=1 -v
+    retryCountOnTaskFailure: 3
+    name: "HubbleConnectivityTests"
+    displayName: "Run Hubble Connectivity Tests"
+
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
       echo $ARTIFACT_DIR

hack/toolbox/server/Dockerfile.heavy

@@ -3,7 +3,7 @@ ADD ./ /
 WORKDIR /
 RUN CGO_ENABLED=0 GOOS=linux go build -o server .
 
-FROM mcr.microsoft.com/oss/mirror/docker.io/library/ubuntu:20.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
 RUN apt-get update
 RUN apt-get install -y \
     axel \
@@ -21,14 +21,14 @@ RUN apt-get install -y \
     net-tools \
     netcat \
     nmap \
-    python \
     python3 \
     ssh \
     sudo \
     tcpdump \
     traceroute \
+    unzip \
     vim \
-    wget
+    wget 
 
 RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
 RUN curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256"

test/integration/hubble/hubble_test.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	k8s "github.com/Azure/azure-container-networking/test/integration"
+	"github.com/Azure/azure-container-networking/test/internal/retry"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+)
+
+const (
+	retryAttempts = 10
+	retryDelay    = 5 * time.Second
+	promAddress   = "http://localhost:9965/metrics"
+	labelSelector = "k8s-app=cilium"
+	namespace     = "kube-system"
+)
+
+var (
+	defaultRetrier  = retry.Retrier{Attempts: retryAttempts, Delay: retryDelay}
+	requiredMetrics = []string{
+		"hubble_flows_processed_total",
+		"hubble_tcp_flags_total",
+	}
+)
+
+func TestEndpoints(t *testing.T) {
+	var kubeconfigPath string
+	flag.StringVar(&kubeconfigPath, "kubeconfig", getDefaultKubeconfigPath(), "Path to the kubeconfig file")
+	flag.Parse()
+
+	config, err := getClientConfig(kubeconfigPath)
+	if err != nil {
+		fmt.Printf("Error creating Kubernetes client config: %v\n", err)
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+	clusterCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+	pingCheckFn := func() error {
+		var pf *k8s.PortForwarder
+		pf, err = k8s.NewPortForwarder(config, t, k8s.PortForwardingOpts{
+			Namespace:     namespace,
+			LabelSelector: labelSelector,
+			LocalPort:     9965,
+			DestPort:      9965,
+		})
+		if err != nil {
+			t.Error(err)
+		}
+		pctx := context.Background()
+
+		portForwardCtx, cancel := context.WithTimeout(pctx, (retryAttempts+1)*retryDelay)
+		defer cancel()
+
+		portForwardFn := func() error {
+			t.Logf("attempting port forward to a pod with label %s, in namespace %s...", labelSelector, namespace)
+			if err := pf.Forward(portForwardCtx); err != nil {
+				return fmt.Errorf("could not start port forward: %w", err)
+			}
+			return nil
+		}
+
+		if err := defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil {
+			t.Fatalf("could not start port forward within %d: %v", (retryAttempts+1)*retryDelay, err)
+		}
+		defer pf.Stop()
+
+		// scrape the hubble metrics
+		metrics, err := getPrometheusMetrics(promAddress)
+		if err != nil {
+			return fmt.Errorf("scraping %s, failed with error: %w", promAddress, err)
+		}
+
+		// verify that the response contains the required metrics
+		for _, reqMetric := range requiredMetrics {
+			if val, exists := metrics[reqMetric]; !exists {
+				return fmt.Errorf("scraping %s, did not find metric %s", val, promAddress) //nolint:goerr113,gocritic
+			}
+		}
+		t.Logf("all metrics validated: %+v", requiredMetrics)
+		return nil
+	}
+
+	if err := defaultRetrier.Do(clusterCtx, pingCheckFn); err != nil {
+		t.Fatalf("metrics check failed with error: %v", err)
+	}
+}
+
+func getPrometheusMetrics(url string) (map[string]struct{}, error) {
+	client := http.Client{}
+	resp, err := client.Get(url) //nolint
+	if err != nil {
+		return nil, fmt.Errorf("HTTP request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP request failed with status: %v", resp.Status) //nolint:goerr113,gocritic
+	}
+
+	metricsData, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading HTTP response body failed: %w", err)
+	}
+
+	metrics := parseMetrics(string(metricsData))
+	return metrics, nil
+}
+
+func parseMetrics(metricsData string) map[string]struct{} {
+	// Create a map to store the strings before the first '{'.
+	metrics := make(map[string]struct{})
+
+	// sample metrics
+	// hubble_tcp_flags_total{destination="",family="IPv4",flag="RST",source="kube-system/metrics-server"} 980
+	// hubble_tcp_flags_total{destination="",family="IPv4",flag="SYN",source="kube-system/ama-metrics"} 1777
+	// we only want the metric name for the time being
+	// label order/parseing can happen later
+	lines := strings.Split(metricsData, "\n")
+	// Iterate through each line.
+	for _, line := range lines {
+		// Find the index of the first '{' character.
+		index := strings.Index(line, "{")
+		if index >= 0 {
+			// Extract the string before the first '{'.
+			str := strings.TrimSpace(line[:index])
+			// Store the string in the map.
+			metrics[str] = struct{}{}
+		}
+	}
+
+	return metrics
+}
+
+func getDefaultKubeconfigPath() string {
+	home := homedir.HomeDir()
+	return filepath.Join(home, ".kube", "config")
+}
+
+func getClientConfig(kubeconfigPath string) (*rest.Config, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		// If running outside a Kubernetes cluster, use the kubeconfig file.
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+		if err != nil {
+			return nil, fmt.Errorf("error creating Kubernetes client config: %w", err)
+		}
+	}
+	return config, nil
+}

test/integration/manifests/cilium/cilium-config-hubble.yaml

@@ -0,0 +1,98 @@
+apiVersion: v1
+data:
+  agent-not-ready-taint-key: node.cilium.io/agent-not-ready
+  arping-refresh-period: 30s
+  auto-direct-node-routes: "false"
+  bpf-lb-external-clusterip: "false"
+  bpf-lb-map-max: "65536"
+  bpf-lb-mode: snat
+  bpf-map-dynamic-size-ratio: "0.0025"
+  bpf-policy-map-max: "16384"
+  bpf-root: /sys/fs/bpf
+  cgroup-root: /run/cilium/cgroupv2
+  cilium-endpoint-gc-interval: 5m0s
+  cluster-id: "0"
+  cluster-name: default
+  debug: "false"
+  disable-cnp-status-updates: "true"
+  disable-endpoint-crd: "false"
+  enable-auto-protect-node-port-range: "true"
+  enable-bgp-control-plane: "false"
+  enable-bpf-clock-probe: "true"
+  enable-endpoint-health-checking: "false"
+  enable-endpoint-routes: "true"
+  enable-health-check-nodeport: "true"
+  enable-health-checking: "true"
+  enable-host-legacy-routing: "true"
+  enable-hubble: "true"
+  enable-ipv4: "true"
+  enable-ipv4-masquerade: "false"
+  enable-ipv6: "false"
+  enable-ipv6-masquerade: "false"
+  enable-k8s-terminating-endpoint: "true"
+  enable-l2-neigh-discovery: "true"
+  enable-l7-proxy: "false"
+  enable-local-node-route: "false"
+  enable-local-redirect-policy: "false"
+  enable-metrics: "true"
+  enable-policy: default
+  enable-remote-node-identity: "true"
+  enable-session-affinity: "true"
+  enable-svc-source-range-check: "true"
+  enable-vtep: "false"
+  enable-well-known-identities: "false"
+  enable-xt-socket-fallback: "true"
+  hubble-metrics: flow:sourceContext=pod-short;destinationContext=pod-short
+    tcp:sourceContext=pod-short;destinationContext=pod-short
+    dns:flow:sourceContext=pod-short;destinationContext=pod-short
+  hubble-metrics-server: :9965
+  hubble-disable-tls: "false"
+  hubble-listen-address: ""
+  hubble-socket-path: /dev/null
+  hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt
+  hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt
+  hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key
+  identity-allocation-mode: crd
+  install-iptables-rules: "true"
+  install-no-conntrack-iptables-rules: "false"
+  ipam: delegated-plugin
+  kube-proxy-replacement: strict
+  kube-proxy-replacement-healthz-bind-address: "0.0.0.0:10256"
+  local-router-ipv4: 169.254.23.0
+  metrics: +cilium_bpf_map_pressure
+  monitor-aggregation: medium
+  monitor-aggregation-flags: all
+  monitor-aggregation-interval: 5s
+  node-port-bind-protection: "true"
+  nodes-gc-interval: 5m0s
+  operator-api-serve-addr: 127.0.0.1:9234
+  operator-prometheus-serve-addr: :9963
+  preallocate-bpf-maps: "false"
+  procfs: /host/proc
+  prometheus-serve-addr: :9962
+  remove-cilium-node-taints: "true"
+  set-cilium-is-up-condition: "true"
+  sidecar-istio-proxy-image: cilium/istio_proxy
+  synchronize-k8s-nodes: "true"
+  tofqdns-dns-reject-response-code: refused
+  tofqdns-enable-dns-compression: "true"
+  tofqdns-endpoint-max-ip-per-hostname: "50"
+  tofqdns-idle-connection-grace-period: 0s
+  tofqdns-max-deferred-connection-deletes: "10000"
+  tofqdns-min-ttl: "3600"
+  tofqdns-proxy-response-max-delay: 100ms
+  tunnel: disabled
+  unmanaged-pod-watcher-interval: "15"
+  vtep-cidr: ""
+  vtep-endpoint: ""
+  vtep-mac: ""
+  vtep-mask: ""
+kind: ConfigMap
+metadata:
+  annotations:
+    meta.helm.sh/release-name: cilium
+    meta.helm.sh/release-namespace: kube-system
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: cilium-config
+  namespace: kube-system

test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: cilium
+  name: hubble-peer
+  namespace: kube-system
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: peer-service
+      port: 443
+      protocol: TCP
+      targetPort: 4244
+  selector:
+    k8s-app: cilium
+  sessionAffinity: None
+  type: ClusterIP