From 78bf146480a60dfbd80d85c70cc3469d4c14a34d Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Tue, 26 Sep 2023 18:18:16 -0500 Subject: [PATCH 1/2] fix: set cluster auto-upgrade and node auto-upgrade in the hackfile (#2253) Signed-off-by: Evan Baker --- hack/aks/Makefile | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 5c3306af95..4aae6f5783 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -8,13 +8,15 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az # overrideable defaults -REGION ?= westus2 +AUTOUPGRADE ?= patch +K8S_VER ?= 1.27 # Used only for ubuntu 18 as K8S 1.24.9, as K8S > 1.25 have Ubuntu 22 +NODE_COUNT ?= 2 +NODEUPGRADE ?= NodeImage OS_SKU ?= Ubuntu -WINDOWS_OS_SKU ?= Windows2022 +OS_SKU_WIN ?= Windows2022 +REGION ?= westus2 VM_SIZE ?= Standard_B2s -NODE_COUNT ?= 2 -K8S_VER ?= 1.27 # Designated for Long Term Support, July 2025 | Only Ubuntu 22.04 is supported -WINDOWS_VM_SKU ?= Standard_B2s +VM_SIZE_WIN ?= Standard_B2s # overrideable variables SUB ?= $(AZURE_SUBSCRIPTION) @@ -88,6 +90,8 @@ up: swift-up ## Alias to swift-up overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -101,6 +105,8 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -115,6 +121,8 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay Cilium cluster overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -128,6 +136,8 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -142,6 +152,8 @@ swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -156,6 +168,8 @@ swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -168,6 +182,8 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster without kube-proxy for Cilium $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -182,6 +198,8 @@ cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster without kube-proxy for Cilium $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -196,6 +214,8 @@ cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster with windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --network-plugin azure \ @@ -207,16 +227,18 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster $(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \ --os-type Windows \ - --os-sku $(WINDOWS_OS_SKU) \ + --os-sku $(OS_SKU_WIN) \ --max-pods 250 \ --name npwin \ --node-count $(NODE_COUNT) \ - -s $(WINDOWS_VM_SKU) + -s $(VM_SIZE_WIN) @$(MAKE) set-kubeconf linux-cniv1-up: rg-up overlay-net-up $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --max-pods 250 \ From 70b691607f67de01e6c029b00d8e80bd9f385d2f Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Thu, 15 Feb 2024 16:29:24 -0600 Subject: [PATCH 2/2] fix: test v1.5 train on k8s 1.28 (#2564) * fix: test main/v1.5 branch on k8s 1.28 Signed-off-by: Evan Baker * 1.28 ds changes * ci: specify vars for envsubst --------- Signed-off-by: Evan Baker Co-authored-by: jpayne3506 --- .pipelines/singletenancy/aks/e2e-job-template.yaml | 1 + .pipelines/templates/create-cluster.yaml | 11 +++++++++-- hack/aks/Makefile | 13 +++++++++++-- .../manifests/cni/cni-installer-v1-windows.yaml | 3 +++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index cbc9cadf57..85502a6600 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -29,6 +29,7 @@ stages: clusterType: ${{ parameters.clusterType }} clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} + vmSizeWin: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index 0066e1ac1e..27b4c42d09 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -5,7 +5,7 @@ parameters: clusterName: "" # Recommended to pass in unique identifier vmSize: "" k8sVersion: "" - windowsOsSku: "Windows2022" # Currently we only support Windows2022 + osSkuWin: "Windows2022" # Currently we only support Windows2022 dependsOn: "" region: "" @@ -31,6 +31,13 @@ jobs: fi mkdir -p ~/.kube/ make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} - make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + + make -C ./hack/aks ${{ parameters.clusterType }} \ + AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \ + CLUSTER=${{ parameters.clusterName }} \ + VM_SIZE=${{ parameters.vmSize }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} \ + OS_SKU_WIN=${{ parameters.osSkuWin }} OS=${{parameters.os}} \ + WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" displayName: Cluster - ${{ parameters.clusterType }} diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 4aae6f5783..c8c12420c4 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -9,7 +9,7 @@ AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v # overrideable defaults AUTOUPGRADE ?= patch -K8S_VER ?= 1.27 # Used only for ubuntu 18 as K8S 1.24.9, as K8S > 1.25 have Ubuntu 22 +K8S_VER ?= 1.27 # Designated for Long Term Support, July 2025 | Only Ubuntu 22.04 is supported NODE_COUNT ?= 2 NODEUPGRADE ?= NodeImage OS_SKU ?= Ubuntu @@ -92,6 +92,7 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -107,6 +108,7 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -123,6 +125,7 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -138,6 +141,7 @@ swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -154,6 +158,7 @@ swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -170,6 +175,7 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -184,6 +190,7 @@ cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -200,6 +207,7 @@ cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster with $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -216,6 +224,7 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --network-plugin azure \ @@ -239,6 +248,7 @@ linux-cniv1-up: rg-up overlay-net-up $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --max-pods 250 \ @@ -247,7 +257,6 @@ linux-cniv1-up: rg-up overlay-net-up --os-sku $(OS_SKU) \ --no-ssh-key \ --yes - @$(MAKE) set-kubeconf down: ## Delete the cluster diff --git a/test/integration/manifests/cni/cni-installer-v1-windows.yaml b/test/integration/manifests/cni/cni-installer-v1-windows.yaml index ca303efff1..07fde1c414 100644 --- a/test/integration/manifests/cni/cni-installer-v1-windows.yaml +++ b/test/integration/manifests/cni/cni-installer-v1-windows.yaml @@ -65,6 +65,9 @@ spec: - azure-vnet-telemetry.config - -o - /k/azurecni/bin/azure-vnet-telemetry.config + env: + - name: PATHEXT + value: .COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL;; volumeMounts: - name: cni-bin mountPath: /k/azurecni/bin/