Skip to content

Commit 846e508

Browse files
authored
ci: [CNI] Adding aks cluster creation steps for k8s e2e test (Azure#2052)
* ci: [CNI] Adding aks cluster creation steps for k8s e2e test * Add validate step to the pipeline * Adding the telemetry config to the cluster
1 parent e9257ed commit 846e508

File tree

10 files changed

+313
-9
lines changed

10 files changed

+313
-9
lines changed

.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ stages:
6666
addSpnToEnvironment: true
6767
inlineScript: |
6868
set -ex
69-
export CNI_IMAGE=$(make acncli-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version))
69+
export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version))
7070
az extension add --name aks-preview
7171
clusterName=${{ parameters.clusterName }}-$(make revision)
7272
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}

.pipelines/pipeline.yaml

+57
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ stages:
3636
echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)"
3737
echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)"
3838
echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)"
39+
echo "##vso[task.setvariable variable=dropgzVersion;isOutput=true]$(make cni-dropgz-version)"
3940
cat /etc/os-release
4041
uname -a
4142
sudo chown -R $(whoami):$(whoami) .
@@ -166,6 +167,16 @@ stages:
166167
arch: amd64
167168
name: cni-dropgz-test
168169
os: linux
170+
cni_plugin_windows2022_amd64:
171+
os: windows
172+
name: cni-plugin
173+
arch: amd64
174+
os_version: ltsc2022
175+
cni_plugin_windows2019_amd64:
176+
os: windows
177+
name: cni-plugin
178+
arch: amd64
179+
os_version: ltsc2019
169180
cns_linux_amd64:
170181
arch: amd64
171182
name: cns
@@ -330,6 +341,48 @@ stages:
330341
clusterName: "swifte2e"
331342
osSku: "Ubuntu"
332343

344+
- template: singletenancy/aks/e2e-job-template.yaml
345+
parameters:
346+
name: "aks_ubuntu_18_04_linux_e2e"
347+
displayName: AKS Ubuntu 18.04
348+
arch: 'amd64'
349+
os: 'linux'
350+
clusterType: linux-cniv1-up
351+
clusterName: 'ubuntu18e2e'
352+
k8sVersion: 1.24.9
353+
354+
- template: singletenancy/aks/e2e-job-template.yaml
355+
parameters:
356+
name: "aks_ubuntu_22_linux_e2e"
357+
displayName: AKS Ubuntu 22
358+
arch: 'amd64'
359+
os: 'linux'
360+
clusterType: linux-cniv1-up
361+
clusterName: 'ubuntu22e2e'
362+
k8sVersion: 1.25
363+
364+
- template: singletenancy/aks/e2e-job-template.yaml
365+
parameters:
366+
name: "aks_windows_19_03_e2e"
367+
displayName: AKS Windows 1903
368+
arch: amd64
369+
os: windows
370+
clusterType: windows-cniv1-up
371+
clusterName: 'win19e2e'
372+
windowsOsSku: 'Windows2019'
373+
os_version: 'ltsc2019'
374+
375+
- template: singletenancy/aks/e2e-job-template.yaml
376+
parameters:
377+
name: "aks_windows_22_e2e"
378+
displayName: AKS Windows 2022
379+
arch: amd64
380+
os: windows
381+
clusterType: windows-cniv1-up
382+
clusterName: 'win22e2e'
383+
windowsOsSku: 'Windows2022'
384+
os_version: 'ltsc2022'
385+
333386
- template: singletenancy/aks-engine/e2e-job-template.yaml
334387
parameters:
335388
name: "ubuntu_18_04_linux_e2e"
@@ -375,6 +428,10 @@ stages:
375428
dependsOn:
376429
- "aks_swift_e2e"
377430
- "cilium_e2e"
431+
- "aks_ubuntu_18_04_linux_e2e"
432+
- "aks_windows_19_03_e2e"
433+
- "aks_ubuntu_22_linux_e2e"
434+
- "aks_windows_22_e2e"
378435
- "ubuntu_18_04_linux_e2e"
379436
- "windows_19_03_e2e"
380437
jobs:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
parameters:
2+
name: ""
3+
displayName: ""
4+
arch: ""
5+
os: ""
6+
clusterType: ""
7+
clusterName: ""
8+
k8sVersion: ""
9+
windowsOsSku: ""
10+
os_version: ""
11+
12+
stages:
13+
- stage: ${{ parameters.name }}
14+
displayName: E2E - ${{ parameters.displayName }}
15+
dependsOn:
16+
- setup
17+
- publish
18+
jobs:
19+
- job: ${{ parameters.name }}
20+
displayName: Singletenancy AKS - (${{ parameters.name }})
21+
pool:
22+
name: $(BUILD_POOL_NAME_DEFAULT)
23+
demands:
24+
- agent.os -equals Linux
25+
- Role -equals $(CUSTOM_E2E_ROLE)
26+
variables:
27+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
28+
GOBIN: "$(GOPATH)/bin" # Go binaries path
29+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
30+
dropgzVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.dropgzVersion'] ]
31+
cniVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ]
32+
steps:
33+
- template: e2e-step-template.yaml
34+
parameters:
35+
name: ${{ parameters.name }}
36+
clusterType: ${{ parameters.clusterType }}
37+
clusterName: ${{ parameters.clusterName }}
38+
arch: ${{ parameters.arch }}
39+
os: ${{ parameters.os }}
40+
k8sVersion: ${{ parameters.k8sVersion }}
41+
windowsOsSku: ${{ parameters.windowsOsSku }}
42+
os_version: ${{ parameters.os_version }}
43+
version: $(dropgzVersion)
44+
cniVersion: $(cniVersion)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
parameters:
2+
name: ""
3+
clusterType: ""
4+
clusterName: ""
5+
nodeCount: ""
6+
vmSize: ""
7+
windowsVMSize: ""
8+
k8sVersion: ""
9+
version: ""
10+
os: ""
11+
windowsOsSku: ""
12+
cniVersion: ""
13+
os_version: ""
14+
15+
steps:
16+
- bash: |
17+
go version
18+
go env
19+
mkdir -p '$(GOBIN)'
20+
mkdir -p '$(GOPATH)/pkg'
21+
mkdir -p '$(modulePath)'
22+
echo '##vso[task.prependpath]$(GOBIN)'
23+
echo '##vso[task.prependpath]$(GOROOT)/bin'
24+
name: "GoEnv"
25+
displayName: "Set up the Go environment"
26+
- task: AzureCLI@1
27+
inputs:
28+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
29+
scriptLocation: "inlineScript"
30+
scriptType: "bash"
31+
addSpnToEnvironment: true
32+
inlineScript: |
33+
mkdir -p ~/.kube/
34+
echo "Create AKS cluster"
35+
echo "parameters ${{ parameters.windowsOsSku }}"
36+
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
37+
make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD}
38+
echo "Cluster successfully created"
39+
displayName: Create test cluster
40+
- script: |
41+
echo "Upload CNI"
42+
if [ "${{parameters.os}}" == "windows" ]; then
43+
export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }})
44+
envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f -
45+
kubectl rollout status daemonset/azure-cni -n kube-system
46+
export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_VERSION=${{ parameters.cniVersion }})
47+
envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
48+
kubectl rollout status daemonset/azure-cni-windows -n kube-system
49+
else
50+
export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }})
51+
envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f -
52+
kubectl rollout status daemonset/azure-cni -n kube-system
53+
fi
54+
name: "UploadCni"
55+
displayName: "Upload CNI"
56+
- task: AzureCLI@1
57+
inputs:
58+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
59+
scriptLocation: "inlineScript"
60+
scriptType: "bash"
61+
addSpnToEnvironment: true
62+
inlineScript: |
63+
clusterName=${{ parameters.clusterName }}-$(make revision)
64+
echo "Restarting nodes"
65+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
66+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
67+
done
68+
displayName: "Restart Nodes"
69+
- script: |
70+
kubectl get pods -A -o wide
71+
echo "Deploying test pods"
72+
cd test/integration/load
73+
go test -count 1 -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=2 -scaleup=40 -os=${{ parameters.os }}
74+
cd ../../..
75+
# Remove this once we have cniv1 support for validating the test cluster
76+
echo "Validate State skipped for linux cniv1 for now"
77+
if [ "${{parameters.os}}" == "windows" ]; then
78+
make test-validate-state OS=${{ parameters.os }}
79+
fi
80+
displayName: "Validate State"
81+
retryCountOnTaskFailure: 3
82+
- task: AzureCLI@1
83+
inputs:
84+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
85+
scriptLocation: "inlineScript"
86+
scriptType: "bash"
87+
addSpnToEnvironment: true
88+
inlineScript: |
89+
echo "Skipping Deleting cluster"
90+
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
91+
make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
92+
echo "Cluster and resources down"
93+
displayName: "Delete test cluster"
94+
condition: always()
95+

Makefile

+7-3
Original file line numberDiff line numberDiff line change
@@ -247,13 +247,15 @@ endif
247247

248248
## Image name definitions.
249249
ACNCLI_IMAGE = acncli
250+
CNI_PLUGIN_IMAGE = azure-cni-plugin
250251
CNI_DROPGZ_IMAGE = cni-dropgz
251252
CNI_DROPGZ_TEST_IMAGE = cni-dropgz-test
252253
CNS_IMAGE = azure-cns
253254
NPM_IMAGE = azure-npm
254255

255256
## Image platform tags.
256257
ACNCLI_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(ACN_VERSION)
258+
CNI_PLUGIN_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNI_VERSION)
257259
CNI_DROPGZ_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNI_DROPGZ_VERSION)
258260
CNI_DROPGZ_TEST_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNI_DROPGZ_TEST_VERSION)
259261
CNS_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNS_VERSION)
@@ -438,15 +440,17 @@ npm-image-pull: ## pull cns container image.
438440
IMAGE=$(NPM_IMAGE) \
439441
TAG=$(NPM_PLATFORM_TAG)
440442

441-
# cni-plugin
443+
# cni-plugin - Specifically used for windows clusters, will be removed once we have Dropgz for windows
444+
cni-plugin-image-name-and-tag: # util target to print the CNI plugin image name and tag.
445+
@echo $(IMAGE_REGISTRY)/$(CNI_PLUGIN_IMAGE):$(CNI_PLUGIN_PLATFORM_TAG)
442446

443447
cni-plugin-image: ## build cni plugin container image.
444448
$(MAKE) container \
445449
DOCKERFILE=cni/build/$(OS).Dockerfile \
446-
IMAGE=$(ACNCLI_IMAGE) \
450+
IMAGE=$(CNI_PLUGIN_IMAGE) \
447451
EXTRA_BUILD_ARGS='--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID) --build-arg OS_VERSION=$(OS_VERSION)' \
448452
PLATFORM=$(PLATFORM) \
449-
TAG=$(ACNCLI_PLATFORM_TAG) \
453+
TAG=$(CNI_PLUGIN_PLATFORM_TAG) \
450454
OS=$(OS) \
451455
ARCH=$(ARCH) \
452456
OS_VERSION=$(OS_VERSION)

cni/build/windows.Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ FROM mcr.microsoft.com/windows/servercore:${OS_VERSION}
1313
SHELL ["powershell", "-command"]
1414
COPY --from=builder /azure-container-networking/azure-vnet.exe azure-vnet.exe
1515
COPY --from=builder /azure-container-networking/azure-vnet-telemetry.exe azure-vnet-telemetry.exe
16+
COPY --from=builder /azure-container-networking/telemetry/azure-vnet-telemetry.config azure-vnet-telemetry.config
1617
COPY --from=builder /azure-container-networking/azure-vnet-ipam.exe azure-vnet-ipam.exe
1718

1819
# This would be replaced with dropgz version of windows.

dropgz/build/cniTest.Dockerfile

+3
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@ WORKDIR /dropgz
1818
COPY dropgz .
1919
COPY --from=azure-ipam /azure-ipam/*.conflist pkg/embed/fs
2020
COPY --from=azure-ipam /azure-ipam/bin/* pkg/embed/fs
21+
COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS.conflist pkg/embed/fs/azure.conflist
2122
COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift.conflist pkg/embed/fs/azure-swift.conflist
2223
COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay.conflist pkg/embed/fs/azure-swift-overlay.conflist
2324
COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay-dualstack.conflist pkg/embed/fs/azure-swift-overlay-dualstack.conflist
2425
COPY --from=azure-vnet /azure-container-networking/bin/* pkg/embed/fs
26+
COPY --from=azure-vnet /azure-container-networking/telemetry/azure-vnet-telemetry.config pkg/embed/fs
27+
2528
RUN cd pkg/embed/fs/ && sha256sum * > sum.txt
2629
RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done
2730

hack/aks/Makefile

+21-5
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ AZIMG = mcr.microsoft.com/azure-cli
88
AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az
99

1010
# overrideable defaults
11-
REGION ?= westus2
12-
OS_SKU ?= Ubuntu
13-
VM_SIZE ?= Standard_B2s
14-
NODE_COUNT ?= 2
11+
REGION ?= westus2
12+
OS_SKU ?= Ubuntu
13+
WINDOWS_OS_SKU ?= Windows2022
14+
VM_SIZE ?= Standard_B2s
15+
NODE_COUNT ?= 2
16+
K8S_VER ?= 1.25 # Used only for ubuntu 18 as K8S 1.24.9, as K8S > 1.25 have Ubuntu 22
17+
WINDOWS_VM_SKU ?= Standard_B2s
1518

1619
# overrideable variables
1720
SUB ?= $(AZURE_SUBSCRIPTION)
@@ -203,14 +206,27 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster
203206

204207
$(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \
205208
--os-type Windows \
206-
--os-sku Windows2022 \
209+
--os-sku $(WINDOWS_OS_SKU) \
207210
--max-pods 250 \
208211
--name npwin \
209212
--node-count $(NODE_COUNT) \
210213
-s $(WINDOWS_VM_SKU)
211214

212215
@$(MAKE) set-kubeconf
213216

217+
linux-cniv1-up: rg-up overlay-net-up
218+
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
219+
--node-count $(NODE_COUNT) \
220+
--node-vm-size $(VM_SIZE) \
221+
--network-plugin azure \
222+
--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
223+
--kubernetes-version $(K8S_VER) \
224+
--os-sku $(OS_SKU) \
225+
--no-ssh-key \
226+
--yes
227+
228+
@$(MAKE) set-kubeconf
229+
214230
down: ## Delete the cluster
215231
$(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes
216232
@$(MAKE) unset-kubeconf

hack/scripts/updatecni.ps1

+5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ Write-Host $env:CONTAINER_SANDBOX_MOUNT_POINT
22
$sourceCNI = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet.exe"
33
$sourceIpam = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-ipam.exe"
44
$sourceTelemetry = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-telemetry.exe"
5+
$sourceTelemetryConfig = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-telemetry.config"
56

67
$sourceCNIVersion = & "$sourceCNI" -v
78
$currentVersion = ""
@@ -36,6 +37,10 @@ if ($currentTelemetryVersion -ne $sourceTelemetryVersion){
3637
Write-Host "copying azure-vnet-telemetry to windows node..."
3738
Remove-Item "C:\k\azurecni\bin\azure-vnet-telemetry.exe"
3839
Copy-Item $sourceTelemetry -Destination "C:\k\azurecni\bin"
40+
41+
Write-Host "copying azure-vnet-telemetry.config to windows node..."
42+
Remove-Item "C:\k\azurecni\bin\azure-vnet-telemetry.config"
43+
Copy-Item $sourceTelemetryConfig -Destination "C:\k\azurecni\bin"
3944
}
4045

4146
## check CNI was already installed so not to get stuck in a infinite loop of rebooting

0 commit comments

Comments
 (0)