diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 04b4c5ad775..10da643ea85 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -378,42 +378,43 @@ blocks: - cd felix - make bin/calico-felix.exe fv/win-fv.exe -- name: "Felix: Windows FV capz" - run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" - dependencies: ["Felix: Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - - export REPORT_DIR=/home/semaphore/report - - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - - export AZURE_TENANT_ID=$AZ_TENANT_ID - - export AZURE_CLIENT_ID=$AZ_SP_ID - - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD - - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" - - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" - - cd felix - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - env_vars: - - name: FV_PROVISIONER - value: "capz" - - name: FV_TYPE - value: "calico-felix" - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - jobs: - - name: CAPZ - Windows FV - commands: - - ./.semaphore/run-win-fv +# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +#- name: "Felix: Windows FV capz" +# run: +# when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" +# dependencies: ["Felix: Build Windows binaries"] +# task: +# secrets: +# - name: banzai-secrets +# - name: private-repo +# prologue: +# commands: +# - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none +# - export REPORT_DIR=/home/semaphore/report +# - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID +# - export AZURE_TENANT_ID=$AZ_TENANT_ID +# - export AZURE_CLIENT_ID=$AZ_SP_ID +# - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD +# - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +# - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" +# - cd felix +# epilogue: +# always: +# commands: +# - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true +# env_vars: +# - name: FV_PROVISIONER +# value: "capz" +# - name: FV_TYPE +# value: "calico-felix" +# - name: SEMAPHORE_ARTIFACT_EXPIRY +# value: 2w +# jobs: +# - name: CAPZ - Windows FV +# commands: +# - ./.semaphore/run-win-fv - name: "Felix: FV Tests" run: diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 746672cacbd..4a68cbf0c1a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -378,42 +378,43 @@ blocks: - cd felix - make bin/calico-felix.exe fv/win-fv.exe -- name: "Felix: Windows FV capz" - run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" - dependencies: ["Felix: Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - - export REPORT_DIR=/home/semaphore/report - - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - - export AZURE_TENANT_ID=$AZ_TENANT_ID - - export AZURE_CLIENT_ID=$AZ_SP_ID - - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD - - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" - - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" - - cd felix - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - env_vars: - - name: FV_PROVISIONER - value: "capz" - - name: FV_TYPE - value: "calico-felix" - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - jobs: - - name: CAPZ - Windows FV - commands: - - ./.semaphore/run-win-fv +# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +#- name: "Felix: Windows FV capz" +# run: +# when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" +# dependencies: ["Felix: Build Windows binaries"] +# task: +# secrets: +# - name: banzai-secrets +# - name: private-repo +# prologue: +# commands: +# - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none +# - export REPORT_DIR=/home/semaphore/report +# - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID +# - export AZURE_TENANT_ID=$AZ_TENANT_ID +# - export AZURE_CLIENT_ID=$AZ_SP_ID +# - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD +# - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +# - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" +# - cd felix +# epilogue: +# always: +# commands: +# - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true +# env_vars: +# - name: FV_PROVISIONER +# value: "capz" +# - name: FV_TYPE +# value: "calico-felix" +# - name: SEMAPHORE_ARTIFACT_EXPIRY +# value: 2w +# jobs: +# - name: CAPZ - Windows FV +# commands: +# - ./.semaphore/run-win-fv - name: "Felix: FV Tests" run: diff --git a/.semaphore/semaphore.yml.d/blocks/20-felix.yml b/.semaphore/semaphore.yml.d/blocks/20-felix.yml index 17345d1835d..e2f0f9327ce 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-felix.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-felix.yml @@ -92,42 +92,43 @@ - cd felix - make bin/calico-felix.exe fv/win-fv.exe -- name: "Felix: Windows FV capz" - run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" - dependencies: ["Felix: Build Windows binaries"] - task: - secrets: - - name: banzai-secrets - - name: private-repo - prologue: - commands: - - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - - export REPORT_DIR=/home/semaphore/report - - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - - export AZURE_TENANT_ID=$AZ_TENANT_ID - - export AZURE_CLIENT_ID=$AZ_SP_ID - - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD - - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" - - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" - - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" - - cd felix - epilogue: - always: - commands: - - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - env_vars: - - name: FV_PROVISIONER - value: "capz" - - name: FV_TYPE - value: "calico-felix" - - name: SEMAPHORE_ARTIFACT_EXPIRY - value: 2w - jobs: - - name: CAPZ - Windows FV - commands: - - ./.semaphore/run-win-fv +# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing. +#- name: "Felix: Windows FV capz" +# run: +# when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" +# dependencies: ["Felix: Build Windows binaries"] +# task: +# secrets: +# - name: banzai-secrets +# - name: private-repo +# prologue: +# commands: +# - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none +# - export REPORT_DIR=/home/semaphore/report +# - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID +# - export AZURE_TENANT_ID=$AZ_TENANT_ID +# - export AZURE_CLIENT_ID=$AZ_SP_ID +# - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD +# - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +# - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')" +# - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')" +# - cd felix +# epilogue: +# always: +# commands: +# - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true +# env_vars: +# - name: FV_PROVISIONER +# value: "capz" +# - name: FV_TYPE +# value: "calico-felix" +# - name: SEMAPHORE_ARTIFACT_EXPIRY +# value: 2w +# jobs: +# - name: CAPZ - Windows FV +# commands: +# - ./.semaphore/run-win-fv - name: "Felix: FV Tests" run: diff --git a/felix/fv/winfv/policy_test.go b/felix/fv/winfv/policy_test.go index 3f6ea99cf56..d4ba5a6a67f 100644 --- a/felix/fv/winfv/policy_test.go +++ b/felix/fv/winfv/policy_test.go @@ -32,13 +32,17 @@ import ( ) func Powershell(args ...string) string { - stdOut, _, err := powershell(args...) + stdOut, stdErr, err := powershell(args...) + if err != nil { + log.Infof("Powershell() error: %s, stdOut: %s, stdErr: %s,", err, stdOut, stdErr) + } ExpectWithOffset(1, err).NotTo(HaveOccurred()) return stdOut } func PowershellWithError(args ...string) string { - _, stdErr, err := powershell(args...) + stdOut, stdErr, err := powershell(args...) + log.Infof("PowershellWithError() error: %s, stdOut: %s, stdErr: %s,", err, stdOut, stdErr) ExpectWithOffset(1, err).To(HaveOccurred()) return stdErr } @@ -59,7 +63,7 @@ func powershell(args ...string) (string, string, error) { err = cmd.Run() if err != nil { - return "", "", err + return stdout.String(), stderr.String(), err } return stdout.String(), stderr.String(), err diff --git a/process/testing/winfv-cni-plugin/aso/export-env.sh b/process/testing/winfv-cni-plugin/aso/export-env.sh index 7831bb21090..dde712ba92a 100755 --- a/process/testing/winfv-cni-plugin/aso/export-env.sh +++ b/process/testing/winfv-cni-plugin/aso/export-env.sh @@ -12,7 +12,22 @@ export AZURE_WINDOWS_IMAGE_VERSION="${AZURE_WINDOWS_IMAGE_VERSION:="17763.5696.2 export LINUX_NODE_COUNT="${LINUX_NODE_COUNT:=1}" export WINDOWS_NODE_COUNT="${WINDOWS_NODE_COUNT:=1}" -export KUBE_VERSION="${KUBE_VERSION:="1.28.7"}" -export CONTAINERD_VERSION="${CONTAINERD_VERSION:="1.6.6"}" + +# Get K8S_VERSION variable from metadata.mk, error out if it cannot be found +SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +METADATAMK=${SCRIPT_CURRENT_DIR}/../../../../metadata.mk +if [ -f ${METADATAMK} ]; then + K8S_VERSION_METADATA=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2) + if [[ ! ${K8S_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Failed to retrieve K8S_VERSION from ${METADATAMK}" + exit 1 + fi +else + echo "Failed to open ${METADATAMK}" + exit 1 +fi +export KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION_METADATA#v}}" + +export CONTAINERD_VERSION="${CONTAINERD_VERSION:="1.6.35"}" export SSH_KEY_FILE="$PWD/.sshkey" diff --git a/process/testing/winfv-felix/capz/Makefile b/process/testing/winfv-felix/capz/Makefile index a893fc7b729..b368b95172e 100644 --- a/process/testing/winfv-felix/capz/Makefile +++ b/process/testing/winfv-felix/capz/Makefile @@ -13,6 +13,7 @@ $(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl @echo "Creating cluster $(CLUSTER_NAME_CAPZ) ..." ./create-cluster.sh $(MAKE) generate-helpers + ./bootstrap-cluster-ips.sh ./replace-win-containerd.sh touch $@ diff --git a/process/testing/winfv-felix/capz/README.md b/process/testing/winfv-felix/capz/README.md index ce83cdc3651..f37ffb6f9bb 100644 --- a/process/testing/winfv-felix/capz/README.md +++ b/process/testing/winfv-felix/capz/README.md @@ -10,11 +10,10 @@ export AZURE_LOCATION="westcentralus" export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_D2s_v3" export AZURE_NODE_MACHINE_TYPE="Standard_D2s_v3" -export KUBE_VERSION="v1.26.6" +export KUBE_VERSION="v1.30.4" export CLUSTER_API_VERSION="v1.5.1" export AZURE_PROVIDER_VERSION="v1.10.4" -export KIND_VERSION="v0.20.0" -export CALICO_VERSION="v3.26.1" +export KIND_VERSION="v0.24.0" # run "az ad sp list --spn your-client-id" to get information. export AZURE_SUBSCRIPTION_ID="" diff --git a/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh b/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh new file mode 100755 index 00000000000..d1a3afd9ae5 --- /dev/null +++ b/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (c) 2024 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +set -e + +: ${KUBECTL:=./bin/kubectl} +: ${KCAPZ:="${KUBECTL} --kubeconfig=./kubeconfig"} +: "${AZURE_RESOURCE_GROUP:?Environment variable empty or not defined.}" + +CAPZ_CONTROL_PLANE_IP=$(az vm list-ip-addresses -g "$AZURE_RESOURCE_GROUP" | jq -r .[0].virtualMachine.network.privateIpAddresses[0]) +echo; echo "Control Plane IP:" "$CAPZ_CONTROL_PLANE_IP" + +vmss_length=$(az vmss list -g "$AZURE_RESOURCE_GROUP" | jq 'length') +LINUX_NAMES="" +WINDOWS_NAMES="" +i=0 +while [[ $i -lt $vmss_length ]]; do + vm=$(az vmss list -g "$AZURE_RESOURCE_GROUP" | jq ".[$i] | .name, .virtualMachineProfile.osProfile.linuxConfiguration.provisionVMAgent, .virtualMachineProfile.osProfile.windowsConfiguration.provisionVMAgent") + name=$(echo $vm | cut -d" " -f1) + is_linux=$(echo $vm | cut -d" " -f2) + if [[ $is_linux == "true" ]]; then + LINUX_NAMES="$LINUX_NAMES $(echo "$name" | tr -d '"')" + fi + is_windows=$(echo $vm | cut -d" " -f3) + if [[ $is_windows == "true" ]]; then + WINDOWS_NAMES="$WINDOWS_NAMES $(echo "$name" | tr -d '"')" + fi + i=$((i+1)) +done + +CAPZ_LINUX_IPS="" +for name in $LINUX_NAMES; do + ip=$(az vmss nic list -g "$AZURE_RESOURCE_GROUP" --vmss-name "$name" | jq -r .[].ipConfigurations[].privateIPAddress) + CAPZ_LINUX_IPS="$CAPZ_LINUX_IPS $ip" +done + +echo; echo "Linux node IPs:" "$CAPZ_LINUX_IPS" + +CAPZ_WINDOWS_IPS="" +for name in $WINDOWS_NAMES; do + ip=$(az vmss nic list -g "$AZURE_RESOURCE_GROUP" --vmss-name "$name" | jq -r .[].ipConfigurations[].privateIPAddress) + CAPZ_WINDOWS_IPS="$CAPZ_WINDOWS_IPS $ip" +done + +echo; echo "Windows node IPs:" "$CAPZ_WINDOWS_IPS" + +if [ ! -f ./ssh-node.sh ]; then + echo "'./ssh-node.sh' helper not found" + exit 1 +fi + +CAPZ_K8S_VERSION_MAJOR=$(${KCAPZ} version -o json | jq -r ".serverVersion.major") +CAPZ_K8S_VERSION_MINOR=$(${KCAPZ} version -o json | jq -r ".serverVersion.minor") + +if [ "$CAPZ_K8S_VERSION_MAJOR" -eq 1 ] && [ "$CAPZ_K8S_VERSION_MINOR" -ge 29 ]; then + echo "In kubernetes v1.29+, kubelet no longer sets up node IPs when using external cloud-provider." + echo "See https://github.com/kubernetes/kubernetes/issues/120720 for more information." +else + echo "Kubernetes version is lower than v1.29, no need for bootstrapping node IPs, exiting" + exit 0 +fi + +echo; echo "Set up node IPs in kubelet config" + +for linux_node_ip in $CAPZ_CONTROL_PLANE_IP $CAPZ_LINUX_IPS; do + ./ssh-node.sh "$linux_node_ip" "cat /etc/default/kubelet" + ./ssh-node.sh "$linux_node_ip" "sudo sh -c 'sed -i -e \"s/\$/ --node-ip=$linux_node_ip/\" /etc/default/kubelet; systemctl restart kubelet'" + ./ssh-node.sh "$linux_node_ip" "cat /etc/default/kubelet" +done + +for windows_node_ip in $CAPZ_WINDOWS_IPS; do + ./ssh-node.sh "$windows_node_ip" "\$file = (get-content C:/k/StartKubelet.ps1); echo \$file" + ./ssh-node.sh "$windows_node_ip" "\$regex = '^\\\$kubeletCommandLine = .*'; \$line = ((get-content C:/k/StartKubelet.ps1) | select-string \$regex); (get-content C:/k/StartKubelet.ps1) -replace \$regex, (\$line.ToString() + ' + \\\" --node-ip=$windows_node_ip\\\"') | set-content c:/k/StartKubelet.ps1; restart-service kubelet" + ./ssh-node.sh "$windows_node_ip" "\$file = (get-content C:/k/StartKubelet.ps1); echo \$file" +done + +echo "Done bootstrapping node IPs" diff --git a/process/testing/winfv-felix/capz/create-cluster.sh b/process/testing/winfv-felix/capz/create-cluster.sh index 97798445796..bfe16d40365 100755 --- a/process/testing/winfv-felix/capz/create-cluster.sh +++ b/process/testing/winfv-felix/capz/create-cluster.sh @@ -105,7 +105,7 @@ sleep 30 # Create a secret to include the password of the Service Principal identity created in Azure # This secret will be referenced by the AzureClusterIdentity used by the AzureCluster -${KUBECTL} create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" +${KUBECTL} create secret generic "${AZURE_CLUSTER_IDENTITY_SECRET_NAME}" --from-literal=clientSecret="${AZURE_CLIENT_SECRET}" --namespace "${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}" # Finally, initialize the management cluster ${CLUSTERCTL} init --infrastructure azure:${AZURE_PROVIDER_VERSION} --core cluster-api:${CLUSTER_API_VERSION} @@ -127,8 +127,12 @@ export AZURE_SSH_PUBLIC_KEY_B64 AZURE_SSH_PUBLIC_KEY=$(< "${SSH_KEY_FILE}.pub" tr -d '\r\n') export AZURE_SSH_PUBLIC_KEY +# Azure image versions use versions corresponding to kubernetes versions, e.g. 129.7.20240717 corresponds to k8s v1.29.7 +AZ_VERSION="$(az vm image list --publisher cncf-upstream --offer capi --all -o json | jq '.[-1].version' -r)" +AZ_KUBE_VERSION="${AZ_VERSION:0:1}"."${AZ_VERSION:1:2}".$(echo "${AZ_VERSION}" | cut -d'.' -f2) + ${CLUSTERCTL} generate cluster ${CLUSTER_NAME_CAPZ} \ - --kubernetes-version ${KUBE_VERSION} \ + --kubernetes-version ${AZ_KUBE_VERSION} \ --control-plane-machine-count=1 \ --worker-machine-count=${WIN_NODE_COUNT}\ --flavor machinepool-windows \ @@ -150,7 +154,7 @@ ${CLUSTERCTL} get kubeconfig ${CLUSTER_NAME_CAPZ} > ./kubeconfig timeout --foreground 600 bash -c "while ! ${KUBECTL} --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 5; done" echo "Cluster config is ready at ./kubeconfig. Run '${KUBECTL} --kubeconfig=./kubeconfig ...' to work with the new target cluster" echo "Waiting for ${TOTAL_NODES} nodes to have been provisioned..." -timeout --foreground 600 bash -c "while ! ${KCAPZ} get nodes | grep ${KUBE_VERSION} | wc -l | grep ${TOTAL_NODES}; do sleep 5; done" +timeout --foreground 600 bash -c "while ! ${KCAPZ} get nodes | grep ${AZ_KUBE_VERSION} | wc -l | grep ${TOTAL_NODES}; do sleep 5; done" echo "Seen all ${TOTAL_NODES} nodes" # Do NOT instal Azure cloud provider (clear taint instead) @@ -159,10 +163,3 @@ retry_command 300 "${KCAPZ} taint nodes --all node.cloudprovider.kubernetes.io/u retry_command 300 "${KCAPZ} taint nodes --selector=!node-role.kubernetes.io/control-plane node.cluster.x-k8s.io/uninitialized:NoSchedule-" echo "Done creating cluster" - -WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) -i=0 -for n in ${WIN_NODES} -do - echo "ID$i: $n"; i=$(expr $i + 1) -done diff --git a/process/testing/winfv-felix/capz/export-env.sh b/process/testing/winfv-felix/capz/export-env.sh index 5cde0b1ba60..9abdd15d400 100755 --- a/process/testing/winfv-felix/capz/export-env.sh +++ b/process/testing/winfv-felix/capz/export-env.sh @@ -16,10 +16,25 @@ export WINDOWS_SERVER_VERSION="${WINDOWS_SERVER_VERSION:="windows-2022"}" export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:="Standard_D2s_v3"}" export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:="Standard_D2s_v3"}" -export KUBE_VERSION=v1.28.9 -export KIND_VERSION=v0.24.0 -export CLUSTER_API_VERSION="${CLUSTER_API_VERSION:="v1.8.1"}" -export AZURE_PROVIDER_VERSION="${AZURE_PROVIDER_VERSION:="v1.13.2"}" -export CONTAINERD_VERSION="${CONTAINERD_VERSION:="v1.7.20"}" -export CALICO_VERSION="${CALICO_VERSION:="v3.28.1"}" +# Retrieve KUBE_VERSION and KIND_VERSION from metadata.mk +SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +METADATAMK=${SCRIPT_CURRENT_DIR}/../../../metadata.mk +if [ -f ${METADATAMK} ]; then + KINDEST_NODE_VERSION_METADATA=$(grep KINDEST_NODE_VERSION ${METADATAMK} | cut -d "=" -f 2) + KIND_VERSION_METADATA=$(grep KIND_VERSION ${METADATAMK} | cut -d "=" -f 2) + if [[ ! ${KINDEST_NODE_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ ! ${KIND_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Failed to retrieve KINDEST_NODE_VERSION and/or KIND_VERSION from ${METADATAMK}" + exit 1 + fi +else + echo "Failed to open ${METADATAMK}" + exit 1 +fi + +export KUBE_VERSION="${KINDEST_NODE_VERSION_METADATA}" +export KIND_VERSION="${KIND_VERSION_METADATA}" + +export CLUSTER_API_VERSION="${CLUSTER_API_VERSION:="v1.7.7"}" +export AZURE_PROVIDER_VERSION="${AZURE_PROVIDER_VERSION:="v1.16.3"}" +export CONTAINERD_VERSION="${CONTAINERD_VERSION:="v1.7.22"}" export YQ_VERSION="${YQ_VERSION:="v4.44.3"}" diff --git a/process/testing/winfv-felix/capz/generate-helpers.sh b/process/testing/winfv-felix/capz/generate-helpers.sh index 2bbb7067752..f5de52a7b8f 100755 --- a/process/testing/winfv-felix/capz/generate-helpers.sh +++ b/process/testing/winfv-felix/capz/generate-helpers.sh @@ -47,34 +47,22 @@ echo echo "Generating helper files" CONNECT_FILE="ssh-node.sh" echo "#---------Connect to Instance--------" | tee ${CONNECT_FILE} -echo "#usage: ./ssh-node.sh 6 to ssh into 10.1.0.6" | tee -a ${CONNECT_FILE} -echo "#usage: ./ssh-node.sh 6 'Get-Service -Name kubelet' > output" | tee -a ${CONNECT_FILE} -echo ssh -t -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@10.1.0.\$1 \$2 | tee -a ${CONNECT_FILE} +echo "#usage: ./ssh-node.sh 10.1.0.6" | tee -a ${CONNECT_FILE} +echo "#usage: ./ssh-node.sh 10.1.0.6 'Get-Service -Name kubelet' > output" | tee -a ${CONNECT_FILE} +echo ssh -t -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@\$1 \$2 | tee -a ${CONNECT_FILE} chmod +x ${CONNECT_FILE} echo SCP_FILE="scp-to-node.sh" echo "#---------Copy files to Instance--------" | tee ${SCP_FILE} -echo "#usage: ./scp-to-node.sh 6 kubeconfig c:\\\\k\\\\kubeconfig -- copy kubeconfig to 10.1.0.6" | tee -a ${SCP_FILE} -echo "#usage: ./scp-to-node.sh 6 images/ebpf-for-windows-c-temp.zip 'c:\\' -- copy temp zip to 10.1.0.6" | tee -a ${SCP_FILE} -echo scp ${OFLAG} -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' \$2 capi@10.1.0.\$1:\$3 | tee -a ${SCP_FILE} +echo "#usage: ./scp-to-node.sh 10.1.0.6 kubeconfig c:\\\\k\\\\kubeconfig -- copy kubeconfig to 10.1.0.6" | tee -a ${SCP_FILE} +echo "#usage: ./scp-to-node.sh 10.1.0.6 images/ebpf-for-windows-c-temp.zip 'c:\\' -- copy temp zip to 10.1.0.6" | tee -a ${SCP_FILE} +echo scp ${OFLAG} -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' \$2 capi@\$1:\$3 | tee -a ${SCP_FILE} chmod +x ${SCP_FILE} echo SCP_FROM_NODE="scp-from-node.sh" echo "#---------Copy files from Instance--------" | tee ${SCP_FROM_NODE} -echo "#usage: ./scp-from-node.sh 6 c:/k/calico.log ./calico.log" | tee -a ${SCP_FROM_NODE} -echo scp ${OFLAG} -r -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@10.1.0.\$1:\$2 \$3 | tee -a ${SCP_FROM_NODE} +echo "#usage: ./scp-from-node.sh 10.1.0.6 c:/k/calico.log ./calico.log" | tee -a ${SCP_FROM_NODE} +echo scp ${OFLAG} -r -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@\$1:\$2 \$3 | tee -a ${SCP_FROM_NODE} chmod +x ${SCP_FROM_NODE} - -# Update env file with Windows ips -sed -i "/^export ID[0-9]=\"[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\"/d" ./export-env.sh - -IP0=`$KCAPZ get node win-p-win000000 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'` -echo; echo "Windows nodes IPs" -echo "IP0: $IP0" - -if [[ $WIN_NODE_COUNT -gt 1 ]]; then - IP1=`$KCAPZ get node win-p-win000001 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'` - echo "IP1: $IP1" -fi diff --git a/process/testing/winfv-felix/capz/replace-win-containerd.ps1 b/process/testing/winfv-felix/capz/replace-win-containerd.ps1 index 9a3f4aef8d0..6ef3ab73075 100644 --- a/process/testing/winfv-felix/capz/replace-win-containerd.ps1 +++ b/process/testing/winfv-felix/capz/replace-win-containerd.ps1 @@ -2,7 +2,7 @@ # Adapted from https://github.com/kubernetes-sigs/windows-testing/blob/e841a06620a293ab2c286b53f562a97f3397595f/capz/templates/windows-base.yaml#L27-L61 Param( - [parameter(Mandatory = $false)] $ContainerdVersion="1.7.13" + [parameter(Mandatory = $false)] $ContainerdVersion="1.7.22" ) $ErrorActionPreference = 'Stop' @@ -41,3 +41,20 @@ if($CONTAINERD_URL -ne ""){ } containerd.exe --version containerd-shim-runhcs-v1.exe --version + +Write-Host Applying registry fix for https://github.com/microsoft/Windows-Containers/issues/516 if on Windows 2022 +$needToRestart = $false +if ((Get-ComputerInfo).OsBuildNumber -eq 20348 -and (Get-ItemProperty -Path "HKLM:SYSTEM\CurrentControlSet\Services\hns\State").FwPerfImprovementChange -ne 0) { + if ((Get-Item -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State') -eq $null ) { + New-Item -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State' + } + Remove-ItemProperty -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State' -Name 'FwPerfImprovementChange' + New-ItemProperty -Path 'HKLM:SYSTEM\\CurrentControlSet\\Services\\hns\\State' -Name 'FwPerfImprovementChange' -Value '0' -PropertyType 'DWORD' + $needToRestart = $true +} + +if ($needToRestart) +{ + Write-host Restarting computer + Restart-Computer -Force +} diff --git a/process/testing/winfv-felix/capz/replace-win-containerd.sh b/process/testing/winfv-felix/capz/replace-win-containerd.sh index d97a0893ee3..e4d5720c426 100755 --- a/process/testing/winfv-felix/capz/replace-win-containerd.sh +++ b/process/testing/winfv-felix/capz/replace-win-containerd.sh @@ -23,7 +23,7 @@ set -o pipefail # access the CAPZ cluster. : ${KUBECTL:=./bin/kubectl} : ${KCAPZ:="${KUBECTL} --kubeconfig=./kubeconfig"} -: ${CONTAINERD_VERSION:="v1.7.13"} +: ${CONTAINERD_VERSION:="v1.7.22"} # Cordon+drain windows nodes, then run the powershell script that replaces # containerd with the specific version wanted and finally uncordon the nodes @@ -32,7 +32,7 @@ echo "Installing containerd ${CONTAINERD_VERSION} on windows nodes" ${KCAPZ} cordon -l kubernetes.io/os=windows ${KCAPZ} drain -l kubernetes.io/os=windows --ignore-daemonsets -WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) +WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | sort) for n in ${WIN_NODES} do ./scp-to-node.sh $n ./replace-win-containerd.ps1 c:\\k\\replace-win-containerd.ps1 diff --git a/process/testing/winfv-felix/setup-fv-capz.sh b/process/testing/winfv-felix/setup-fv-capz.sh index 50001025b24..c129ca87c6b 100644 --- a/process/testing/winfv-felix/setup-fv-capz.sh +++ b/process/testing/winfv-felix/setup-fv-capz.sh @@ -83,17 +83,20 @@ function start_cluster(){ fi # Use EXIT_CODE to bypass errexit and capture more information about a possible failure here EXIT_CODE=0 - make -C $CAPZ_LOCATION install-calico RELEASE_STREAM=local HASH_RELEASE=true PRODUCT=calico || EXIT_CODE=$? + make -C $CAPZ_LOCATION install-calico RELEASE_STREAM=master HASH_RELEASE=true PRODUCT=calico || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]]; then echo "failed to install Calico" ${KCAPZ} describe tigerastatus ${KCAPZ} get tigerastatus -o yaml exit $EXIT_CODE fi + + # Enable felix debug logging + ${KCAPZ} patch felixconfiguration default --type merge --patch='{"spec":{"logSeverityScreen":"Debug"}}' + #Get Windows node ip - WIN_NODE=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk -v OFS='\t\t' '{print $6}') - export WIN_NODE_IP=${WIN_NODE: -1} - export LINUX_NODE=$(${KCAPZ} get nodes -l kubernetes.io/os=linux,'!node-role.kubernetes.io/control-plane' -o wide --no-headers | awk -v OFS='\t\t' '{print $6}') + export WIN_NODE_IP=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk -v OFS='\t\t' '{print $6}') + export LINUX_NODE_IP=$(${KCAPZ} get nodes -l kubernetes.io/os=linux,'!node-role.kubernetes.io/control-plane' -o wide --no-headers | awk -v OFS='\t\t' '{print $6}') } function upload_calico_images(){ @@ -134,7 +137,7 @@ function prepare_fv(){ FV_RUN_CNI=$CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1 cp $CALICO_HOME/process/testing/winfv-cni-plugin/run-fv-cni-plugin.ps1 $FV_RUN_CNI sed -i "s??${KUBE_VERSION}?g" $FV_RUN_CNI - sed -i "s??${LINUX_NODE}?g" $FV_RUN_CNI + sed -i "s??${LINUX_NODE_IP}?g" $FV_RUN_CNI sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_CNI sed -i "s??containerd?g" $FV_RUN_CNI sed -i "s??${CONTAINERD_VERSION}?g" $FV_RUN_CNI @@ -144,7 +147,7 @@ function prepare_fv(){ FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1 cp $CALICO_HOME/process/testing/winfv-felix/run-fv-full.ps1 $FV_RUN_FELIX sed -i "s??${KUBE_VERSION}?g" $FV_RUN_FELIX - sed -i "s??${LINUX_NODE}?g" $FV_RUN_FELIX + sed -i "s??${LINUX_NODE_IP}?g" $FV_RUN_FELIX sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_FELIX sed -i "s??containerd?g" $FV_RUN_FELIX sed -i "s??${CONTAINERD_VERSION}?g" $FV_RUN_FELIX @@ -202,10 +205,20 @@ function run_windows_fv(){ } function get_test_results(){ + # Get test logs $CAPZ_LOCATION/scp-from-node.sh $WIN_NODE_IP c:\\k\\report\\* $REPORT_DIR if [[ $SEMAPHORE == "false" ]]; then cat $REPORT_DIR/fv-test.log fi + + # Get logs from windows pod + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c uninstall-calico > $REPORT_DIR/win-uninstall-calico.log + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c install-cni > $REPORT_DIR/win-install-cni.log + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c node > $REPORT_DIR/win-node.log + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node-windows -c felix > $REPORT_DIR/win-felix.log + + # Get logs from linux pod + ${KCAPZ} logs -n calico-system -l k8s-app=calico-node -c calico-node > $REPORT_DIR/linux-calico-node.log } prepare_env diff --git a/process/testing/winfv-felix/setup-fv.sh b/process/testing/winfv-felix/setup-fv.sh index 98a643f647d..ca3831aa815 100755 --- a/process/testing/winfv-felix/setup-fv.sh +++ b/process/testing/winfv-felix/setup-fv.sh @@ -16,17 +16,22 @@ # Prefix for cluster name NAME_PREFIX="${NAME_PREFIX:=${USER}-win-fv}" -# Get K8S_VERSION variable from metadata.mk, default to a value if it cannot be found +# Get K8S_VERSION variable from metadata.mk, error out if it cannot be found SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" METADATAMK=${SCRIPT_CURRENT_DIR}/../../../metadata.mk if [ -f ${METADATAMK} ]; then - K8S_VERSION=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2) + K8S_VERSION_METADATA=$(grep K8S_VERSION ${METADATAMK} | cut -d "=" -f 2) + if [[ ! ${K8S_VERSION_METADATA} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Failed to retrieve K8S_VERSION from ${METADATAMK}" + exit 1 + fi else - K8S_VERSION=v1.27.11 + echo "Failed to open ${METADATAMK}" + exit 1 fi # Kubernetes version -KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION#v}}" +KUBE_VERSION="${KUBE_VERSION:=${K8S_VERSION_METADATA#v}}" # AWS keypair name for nodes WINDOWS_KEYPAIR_NAME="${WINDOWS_KEYPAIR_NAME:=AWS-key-pair}" @@ -54,7 +59,7 @@ WINDOWS_OS="${WINDOWS_OS:=Windows2022container}" CONTAINER_RUNTIME="${CONTAINER_RUNTIME:=docker}" # Specify containerd version to use. -CONTAINERD_VERSION="${CONTAINERD_VERSION:=1.6.22}" +CONTAINERD_VERSION="${CONTAINERD_VERSION:=1.6.35}" #specify description of AMI,this would be a filter to search AMI. #we can create a Json file for description and use it. TODO???