diff --git a/process/testing/winfv-felix/capz/Makefile b/process/testing/winfv-felix/capz/Makefile index a893fc7b729..b368b95172e 100644 --- a/process/testing/winfv-felix/capz/Makefile +++ b/process/testing/winfv-felix/capz/Makefile @@ -13,6 +13,7 @@ $(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl @echo "Creating cluster $(CLUSTER_NAME_CAPZ) ..." ./create-cluster.sh $(MAKE) generate-helpers + ./bootstrap-cluster-ips.sh ./replace-win-containerd.sh touch $@ diff --git a/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh b/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh new file mode 100755 index 00000000000..d1a3afd9ae5 --- /dev/null +++ b/process/testing/winfv-felix/capz/bootstrap-cluster-ips.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (c) 2024 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +set -e + +: ${KUBECTL:=./bin/kubectl} +: ${KCAPZ:="${KUBECTL} --kubeconfig=./kubeconfig"} +: "${AZURE_RESOURCE_GROUP:?Environment variable empty or not defined.}" + +CAPZ_CONTROL_PLANE_IP=$(az vm list-ip-addresses -g "$AZURE_RESOURCE_GROUP" | jq -r .[0].virtualMachine.network.privateIpAddresses[0]) +echo; echo "Control Plane IP:" "$CAPZ_CONTROL_PLANE_IP" + +vmss_length=$(az vmss list -g "$AZURE_RESOURCE_GROUP" | jq 'length') +LINUX_NAMES="" +WINDOWS_NAMES="" +i=0 +while [[ $i -lt $vmss_length ]]; do + vm=$(az vmss list -g "$AZURE_RESOURCE_GROUP" | jq ".[$i] | .name, .virtualMachineProfile.osProfile.linuxConfiguration.provisionVMAgent, .virtualMachineProfile.osProfile.windowsConfiguration.provisionVMAgent") + name=$(echo $vm | cut -d" " -f1) + is_linux=$(echo $vm | cut -d" " -f2) + if [[ $is_linux == "true" ]]; then + LINUX_NAMES="$LINUX_NAMES $(echo "$name" | tr -d '"')" + fi + is_windows=$(echo $vm | cut -d" " -f3) + if [[ $is_windows == "true" ]]; then + WINDOWS_NAMES="$WINDOWS_NAMES $(echo "$name" | tr -d '"')" + fi + i=$((i+1)) +done + +CAPZ_LINUX_IPS="" +for name in $LINUX_NAMES; do + ip=$(az vmss nic list -g "$AZURE_RESOURCE_GROUP" --vmss-name "$name" | jq -r .[].ipConfigurations[].privateIPAddress) + CAPZ_LINUX_IPS="$CAPZ_LINUX_IPS $ip" +done + +echo; echo "Linux node IPs:" "$CAPZ_LINUX_IPS" + +CAPZ_WINDOWS_IPS="" +for name in $WINDOWS_NAMES; do + ip=$(az vmss nic list -g "$AZURE_RESOURCE_GROUP" --vmss-name "$name" | jq -r .[].ipConfigurations[].privateIPAddress) + CAPZ_WINDOWS_IPS="$CAPZ_WINDOWS_IPS $ip" +done + +echo; echo "Windows node IPs:" "$CAPZ_WINDOWS_IPS" + +if [ ! -f ./ssh-node.sh ]; then + echo "'./ssh-node.sh' helper not found" + exit 1 +fi + +CAPZ_K8S_VERSION_MAJOR=$(${KCAPZ} version -o json | jq -r ".serverVersion.major") +CAPZ_K8S_VERSION_MINOR=$(${KCAPZ} version -o json | jq -r ".serverVersion.minor") + +if [ "$CAPZ_K8S_VERSION_MAJOR" -eq 1 ] && [ "$CAPZ_K8S_VERSION_MINOR" -ge 29 ]; then + echo "In kubernetes v1.29+, kubelet no longer sets up node IPs when using external cloud-provider." + echo "See https://github.com/kubernetes/kubernetes/issues/120720 for more information." +else + echo "Kubernetes version is lower than v1.29, no need for bootstrapping node IPs, exiting" + exit 0 +fi + +echo; echo "Set up node IPs in kubelet config" + +for linux_node_ip in $CAPZ_CONTROL_PLANE_IP $CAPZ_LINUX_IPS; do + ./ssh-node.sh "$linux_node_ip" "cat /etc/default/kubelet" + ./ssh-node.sh "$linux_node_ip" "sudo sh -c 'sed -i -e \"s/\$/ --node-ip=$linux_node_ip/\" /etc/default/kubelet; systemctl restart kubelet'" + ./ssh-node.sh "$linux_node_ip" "cat /etc/default/kubelet" +done + +for windows_node_ip in $CAPZ_WINDOWS_IPS; do + ./ssh-node.sh "$windows_node_ip" "\$file = (get-content C:/k/StartKubelet.ps1); echo \$file" + ./ssh-node.sh "$windows_node_ip" "\$regex = '^\\\$kubeletCommandLine = .*'; \$line = ((get-content C:/k/StartKubelet.ps1) | select-string \$regex); (get-content C:/k/StartKubelet.ps1) -replace \$regex, (\$line.ToString() + ' + \\\" --node-ip=$windows_node_ip\\\"') | set-content c:/k/StartKubelet.ps1; restart-service kubelet" + ./ssh-node.sh "$windows_node_ip" "\$file = (get-content C:/k/StartKubelet.ps1); echo \$file" +done + +echo "Done bootstrapping node IPs" diff --git a/process/testing/winfv-felix/capz/create-cluster.sh b/process/testing/winfv-felix/capz/create-cluster.sh index 4ffd16980ef..f9004b2fd08 100755 --- a/process/testing/winfv-felix/capz/create-cluster.sh +++ b/process/testing/winfv-felix/capz/create-cluster.sh @@ -172,10 +172,3 @@ retry_command 300 "${KCAPZ} taint nodes --all node.cloudprovider.kubernetes.io/u retry_command 300 "${KCAPZ} taint nodes --selector=!node-role.kubernetes.io/control-plane node.cluster.x-k8s.io/uninitialized:NoSchedule-" echo "Done creating cluster" - -WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) -i=0 -for n in ${WIN_NODES} -do - echo "ID$i: $n"; i=$(expr $i + 1) -done diff --git a/process/testing/winfv-felix/capz/generate-helpers.sh b/process/testing/winfv-felix/capz/generate-helpers.sh index 2bbb7067752..f5de52a7b8f 100755 --- a/process/testing/winfv-felix/capz/generate-helpers.sh +++ b/process/testing/winfv-felix/capz/generate-helpers.sh @@ -47,34 +47,22 @@ echo echo "Generating helper files" CONNECT_FILE="ssh-node.sh" echo "#---------Connect to Instance--------" | tee ${CONNECT_FILE} -echo "#usage: ./ssh-node.sh 6 to ssh into 10.1.0.6" | tee -a ${CONNECT_FILE} -echo "#usage: ./ssh-node.sh 6 'Get-Service -Name kubelet' > output" | tee -a ${CONNECT_FILE} -echo ssh -t -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@10.1.0.\$1 \$2 | tee -a ${CONNECT_FILE} +echo "#usage: ./ssh-node.sh 10.1.0.6" | tee -a ${CONNECT_FILE} +echo "#usage: ./ssh-node.sh 10.1.0.6 'Get-Service -Name kubelet' > output" | tee -a ${CONNECT_FILE} +echo ssh -t -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@\$1 \$2 | tee -a ${CONNECT_FILE} chmod +x ${CONNECT_FILE} echo SCP_FILE="scp-to-node.sh" echo "#---------Copy files to Instance--------" | tee ${SCP_FILE} -echo "#usage: ./scp-to-node.sh 6 kubeconfig c:\\\\k\\\\kubeconfig -- copy kubeconfig to 10.1.0.6" | tee -a ${SCP_FILE} -echo "#usage: ./scp-to-node.sh 6 images/ebpf-for-windows-c-temp.zip 'c:\\' -- copy temp zip to 10.1.0.6" | tee -a ${SCP_FILE} -echo scp ${OFLAG} -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' \$2 capi@10.1.0.\$1:\$3 | tee -a ${SCP_FILE} +echo "#usage: ./scp-to-node.sh 10.1.0.6 kubeconfig c:\\\\k\\\\kubeconfig -- copy kubeconfig to 10.1.0.6" | tee -a ${SCP_FILE} +echo "#usage: ./scp-to-node.sh 10.1.0.6 images/ebpf-for-windows-c-temp.zip 'c:\\' -- copy temp zip to 10.1.0.6" | tee -a ${SCP_FILE} +echo scp ${OFLAG} -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' \$2 capi@\$1:\$3 | tee -a ${SCP_FILE} chmod +x ${SCP_FILE} echo SCP_FROM_NODE="scp-from-node.sh" echo "#---------Copy files from Instance--------" | tee ${SCP_FROM_NODE} -echo "#usage: ./scp-from-node.sh 6 c:/k/calico.log ./calico.log" | tee -a ${SCP_FROM_NODE} -echo scp ${OFLAG} -r -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@10.1.0.\$1:\$2 \$3 | tee -a ${SCP_FROM_NODE} +echo "#usage: ./scp-from-node.sh 10.1.0.6 c:/k/calico.log ./calico.log" | tee -a ${SCP_FROM_NODE} +echo scp ${OFLAG} -r -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o \'ProxyCommand ssh -i ${LOCAL_PATH}/.sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p capi@${APISERVER}\' capi@\$1:\$2 \$3 | tee -a ${SCP_FROM_NODE} chmod +x ${SCP_FROM_NODE} - -# Update env file with Windows ips -sed -i "/^export ID[0-9]=\"[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\"/d" ./export-env.sh - -IP0=`$KCAPZ get node win-p-win000000 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'` -echo; echo "Windows nodes IPs" -echo "IP0: $IP0" - -if [[ $WIN_NODE_COUNT -gt 1 ]]; then - IP1=`$KCAPZ get node win-p-win000001 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'` - echo "IP1: $IP1" -fi diff --git a/process/testing/winfv-felix/capz/replace-win-containerd.sh b/process/testing/winfv-felix/capz/replace-win-containerd.sh index 03c3a33f02a..e4d5720c426 100755 --- a/process/testing/winfv-felix/capz/replace-win-containerd.sh +++ b/process/testing/winfv-felix/capz/replace-win-containerd.sh @@ -32,7 +32,7 @@ echo "Installing containerd ${CONTAINERD_VERSION} on windows nodes" ${KCAPZ} cordon -l kubernetes.io/os=windows ${KCAPZ} drain -l kubernetes.io/os=windows --ignore-daemonsets -WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) +WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | sort) for n in ${WIN_NODES} do ./scp-to-node.sh $n ./replace-win-containerd.ps1 c:\\k\\replace-win-containerd.ps1 diff --git a/process/testing/winfv-felix/setup-fv-capz.sh b/process/testing/winfv-felix/setup-fv-capz.sh index 7f0f092457b..bd1589639e8 100644 --- a/process/testing/winfv-felix/setup-fv-capz.sh +++ b/process/testing/winfv-felix/setup-fv-capz.sh @@ -91,9 +91,8 @@ function start_cluster(){ exit $EXIT_CODE fi #Get Windows node ip - WIN_NODE=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk -v OFS='\t\t' '{print $6}') - export WIN_NODE_IP=${WIN_NODE: -1} - export LINUX_NODE=$(${KCAPZ} get nodes -l kubernetes.io/os=linux,'!node-role.kubernetes.io/control-plane' -o wide --no-headers | awk -v OFS='\t\t' '{print $6}') + WIN_NODE_IP=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk -v OFS='\t\t' '{print $6}') + export LINUX_NODE_IP=$(${KCAPZ} get nodes -l kubernetes.io/os=linux,'!node-role.kubernetes.io/control-plane' -o wide --no-headers | awk -v OFS='\t\t' '{print $6}') } function upload_calico_images(){ @@ -134,7 +133,7 @@ function prepare_fv(){ FV_RUN_CNI=$CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1 cp $CALICO_HOME/process/testing/winfv-cni-plugin/run-fv-cni-plugin.ps1 $FV_RUN_CNI sed -i "s??${KUBE_VERSION}?g" $FV_RUN_CNI - sed -i "s??${LINUX_NODE}?g" $FV_RUN_CNI + sed -i "s??${LINUX_NODE_IP}?g" $FV_RUN_CNI sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_CNI sed -i "s??containerd?g" $FV_RUN_CNI sed -i "s??${CONTAINERD_VERSION}?g" $FV_RUN_CNI @@ -144,7 +143,7 @@ function prepare_fv(){ FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1 cp $CALICO_HOME/process/testing/winfv-felix/run-fv-full.ps1 $FV_RUN_FELIX sed -i "s??${KUBE_VERSION}?g" $FV_RUN_FELIX - sed -i "s??${LINUX_NODE}?g" $FV_RUN_FELIX + sed -i "s??${LINUX_NODE_IP}?g" $FV_RUN_FELIX sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_FELIX sed -i "s??containerd?g" $FV_RUN_FELIX sed -i "s??${CONTAINERD_VERSION}?g" $FV_RUN_FELIX