CAPZ Win FV improvements

Use master hashrel build for Win FVs.

Use k8s and kind versions from metadata.mk in Win FVs.

Extract latest KUBE_VERSION from az images to use in capz cluster (as they
might not exactly match the versions from metadata.mk).

Bump capz versions.

Add node IP bootstrapping on k8s v1.29+ (as kubelet no longer sets node IPs on external cloud-providers).

Change generated ssh/scp helpers to use full node IPs.

Enable felix debug logging and collect pod logs at the end of tests.

Add more logging on powershell commands in windows policy_test.go

Add workaround for microsoft/Windows-Containers#516 to CAPZ Win FVs.

Disable Felix CAPZ Windows FVs temporarily.

.semaphore/semaphore.yml.d/blocks/20-felix.yml

@@ -92,42 +92,43 @@
           - cd felix
           - make bin/calico-felix.exe fv/win-fv.exe
 
-- name: "Felix: Windows FV capz"
-  run:
-    when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
-  dependencies: ["Felix: Build Windows binaries"]
-  task:
-    secrets:
-      - name: banzai-secrets
-      - name: private-repo
-    prologue:
-      commands:
-        - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none
-        - export REPORT_DIR=/home/semaphore/report
-        - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID
-        - export AZURE_TENANT_ID=$AZ_TENANT_ID
-        - export AZURE_CLIENT_ID=$AZ_SP_ID
-        - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD
-        - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
-        - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')"
-        - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')"
-        - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')"
-        - cd felix
-    epilogue:
-      always:
-        commands:
-          - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
-    env_vars:
-      - name: FV_PROVISIONER
-        value: "capz"
-      - name: FV_TYPE
-        value: "calico-felix"
-      - name: SEMAPHORE_ARTIFACT_EXPIRY
-        value: 2w
-    jobs:
-      - name: CAPZ - Windows FV
-        commands:
-          - ./.semaphore/run-win-fv
+# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing.
+#- name: "Felix: Windows FV capz"
+#  run:
+#    when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
+#  dependencies: ["Felix: Build Windows binaries"]
+#  task:
+#    secrets:
+#      - name: banzai-secrets
+#      - name: private-repo
+#    prologue:
+#      commands:
+#        - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none
+#        - export REPORT_DIR=/home/semaphore/report
+#        - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID
+#        - export AZURE_TENANT_ID=$AZ_TENANT_ID
+#        - export AZURE_CLIENT_ID=$AZ_SP_ID
+#        - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD
+#        - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
+#        - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')"
+#        - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')"
+#        - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')"
+#        - cd felix
+#    epilogue:
+#      always:
+#        commands:
+#          - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
+#    env_vars:
+#      - name: FV_PROVISIONER
+#        value: "capz"
+#      - name: FV_TYPE
+#        value: "calico-felix"
+#      - name: SEMAPHORE_ARTIFACT_EXPIRY
+#        value: 2w
+#    jobs:
+#      - name: CAPZ - Windows FV
+#        commands:
+#          - ./.semaphore/run-win-fv
 
 - name: "Felix: FV Tests"
   run:

felix/fv/winfv/policy_test.go

@@ -32,13 +32,17 @@ import (
 )
 
 func Powershell(args ...string) string {
-	stdOut, _, err := powershell(args...)
+	stdOut, stdErr, err := powershell(args...)
+	if err != nil {
+		log.Infof("Powershell() error: %s, stdOut: %s, stdErr: %s,", err, stdOut, stdErr)
+	}
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 	return stdOut
 }
 
 func PowershellWithError(args ...string) string {
-	_, stdErr, err := powershell(args...)
+	stdOut, stdErr, err := powershell(args...)
+	log.Infof("PowershellWithError() error: %s, stdOut: %s, stdErr: %s,", err, stdOut, stdErr)
 	ExpectWithOffset(1, err).To(HaveOccurred())
 	return stdErr
 }
@@ -59,7 +63,7 @@ func powershell(args ...string) (string, string, error) {
 
 	err = cmd.Run()
 	if err != nil {
-		return "", "", err
+		return stdout.String(), stderr.String(), err
 	}
 
 	return stdout.String(), stderr.String(), err

process/testing/winfv-cni-plugin/aso/export-env.sh

@@ -12,7 +12,22 @@ export AZURE_WINDOWS_IMAGE_VERSION="${AZURE_WINDOWS_IMAGE_VERSION:="17763.5696.2
 export LINUX_NODE_COUNT="${LINUX_NODE_COUNT:=1}"
 export WINDOWS_NODE_COUNT="${WINDOWS_NODE_COUNT:=1}"
 
-export KUBE_VERSION="${KUBE_VERSION:="1.28.7"}"
-export CONTAINERD_VERSION="${CONTAINERD_VERSION:="1.6.6"}"
+
+# Get K8S_VERSION variable from metadata.mk, error out if it cannot be found
+SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+METADATAMK=${SCRIPT_CURRENT_DIR}/../../../../metadata.mk
+if [ -f ${METADATAMK} ]; then
+    K8S_VERSION_METADATA=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2)
+    if [[ ! ${K8S_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "Failed to retrieve K8S_VERSION from ${METADATAMK}"
+        exit 1
+    fi
+else
+    echo "Failed to open ${METADATAMK}"
+    exit 1
+fi
+export KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION_METADATA#v}}"
+
+export CONTAINERD_VERSION="${CONTAINERD_VERSION:="1.6.35"}"
 
 export SSH_KEY_FILE="$PWD/.sshkey"

process/testing/winfv-felix/capz/Makefile

@@ -13,6 +13,7 @@ $(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl
 	@echo "Creating cluster $(CLUSTER_NAME_CAPZ) ..."
 	./create-cluster.sh
 	$(MAKE) generate-helpers
+	./bootstrap-cluster-ips.sh
 	./replace-win-containerd.sh
 	touch $@
 

process/testing/winfv-felix/capz/README.md

@@ -10,11 +10,10 @@ export AZURE_LOCATION="westcentralus"
 export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_D2s_v3"
 export AZURE_NODE_MACHINE_TYPE="Standard_D2s_v3"
 
-export KUBE_VERSION="v1.26.6"
+export KUBE_VERSION="v1.30.4"
 export CLUSTER_API_VERSION="v1.5.1"
 export AZURE_PROVIDER_VERSION="v1.10.4"
-export KIND_VERSION="v0.20.0"
-export CALICO_VERSION="v3.26.1"
+export KIND_VERSION="v0.24.0"
 
 # run "az ad sp list --spn your-client-id" to get information.
 export AZURE_SUBSCRIPTION_ID="<your subscription id>"