Skip to content

Commit b59476a

Browse files
jpayne3506jpayne3506
committed
ci:[CNI] Replace AKS-Engine Tests with k8s conformance tests (#2062)
* Initial Commit * Add attempts to prevent flakyness * Add taint for windows tests * Add k8s e2e tests * Testing vmSizes * Artifact k8se2e binary * Remove NPM E2E * Add testing and increase processes * Addressing comments (cherry picked from commit 451c691)
1 parent c1b39e8 commit b59476a

11 files changed

+371
-83
lines changed

‎.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
parameters:
2+
clusterName: ""
3+
os: ""
4+
dependsOn: ""
5+
sub: ""
6+
7+
8+
jobs:
9+
- job: cni_k8se2e
10+
displayName: "CNI k8s E2E"
11+
dependsOn: ${{ parameters.dependsOn }}
12+
pool:
13+
name: $(BUILD_POOL_NAME_DEFAULT)
14+
steps:
15+
- task: AzureCLI@1
16+
inputs:
17+
azureSubscription: ${{ parameters.sub }}
18+
scriptLocation: "inlineScript"
19+
scriptType: "bash"
20+
addSpnToEnvironment: true
21+
inlineScript: |
22+
set -e
23+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
24+
25+
# sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx"
26+
# pulling k8s version from AKS.
27+
eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }}-$(make revision) -n ${{ parameters.clusterName }}-$(make revision) --query "currentKubernetesVersion")
28+
curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz
29+
30+
# https://github.com/kubernetes/sig-release/blob/master/release-engineering/artifacts.md#content-of-kubernetes-test-system-archtargz-on-example-of-kubernetes-test-linux-amd64targz-directories-removed-from-list
31+
# explictly unzip and strip directories from ginkgo and e2e.test
32+
tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
33+
34+
displayName: "Setup Environment"
35+
- ${{ if eq(parameters.datapath, true) }}:
36+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
37+
parameters:
38+
testName: Datapath
39+
name: datapath
40+
clusterName: ${{ parameters.clusterName }}
41+
ginkgoFocus: '(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api'
42+
ginkgoSkip: 'SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6'
43+
os: ${{ parameters.os }}
44+
processes: 8
45+
attempts: 10
46+
- ${{ if eq(parameters.dns, true) }}:
47+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
48+
parameters:
49+
testName: DNS
50+
name: dns
51+
clusterName: ${{ parameters.clusterName }}
52+
ginkgoFocus: '\[sig-network\].DNS.should'
53+
ginkgoSkip: 'resolv'
54+
os: ${{ parameters.os }}
55+
processes: 8
56+
attempts: 3
57+
- ${{ if eq(parameters.portforward, true) }}:
58+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
59+
parameters:
60+
testName: Kubectl Portforward
61+
name: portforward
62+
clusterName: ${{ parameters.clusterName }}
63+
ginkgoFocus: '\[sig-cli\].Kubectl.Port'
64+
ginkgoSkip: ''
65+
os: ${{ parameters.os }}
66+
processes: 8
67+
attempts: 3
68+
- ${{ if eq(parameters.loadBalancer, true) }}:
69+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
70+
parameters:
71+
testName: Load Balancers
72+
name: load
73+
clusterName: ${{ parameters.clusterName }}
74+
ginkgoFocus: '\[sig-network\].LoadBalancers'
75+
ginkgoSkip: 'ESIPP|Serial'
76+
os: ${{ parameters.os }}
77+
processes: 8
78+
attempts: 3
79+
- ${{ if eq(parameters.service, true) }}:
80+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
81+
parameters:
82+
testName: Service Conformance
83+
name: service
84+
clusterName: ${{ parameters.clusterName }}
85+
ginkgoFocus: 'Services.*\[Conformance\].*'
86+
ginkgoSkip: ''
87+
os: ${{ parameters.os }}
88+
processes: 8
89+
attempts: 3
90+
- ${{ if eq(parameters.hostport, true) }}:
91+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
92+
parameters:
93+
testName: Host Port
94+
name: hostport
95+
clusterName: ${{ parameters.clusterName }}
96+
ginkgoFocus: '\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort'
97+
ginkgoSkip: 'SCTP|exists conflict' # Skip slow 5 minute test
98+
os: ${{ parameters.os }}
99+
processes: 1 # Has a short serial test
100+
attempts: 3
101+
- ${{ if and(eq(parameters.hybridWin, true), eq(parameters.os, 'windows')) }}:
102+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
103+
parameters:
104+
testName: Hybrid Network
105+
name: hybrid
106+
clusterName: ${{ parameters.clusterName }}
107+
ginkgoFocus: '\[sig-windows\].Hybrid'
108+
ginkgoSkip: ''
109+
os: ${{ parameters.os }}
110+
processes: 8
111+
attempts: 3

‎.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
parameters:
2+
testName: ""
3+
name: ""
4+
clusterName: ""
5+
ginkgoFocus: ""
6+
ginkgoSkip: ""
7+
os: ""
8+
processes: "" # Number of parallel processes
9+
attempts: ""
10+
11+
12+
steps:
13+
- script: |
14+
set -ex
15+
16+
# ginkgoSkip cant handle only |LinuxOnly. Need to have check
17+
if ${{ lower(and(ge(length(parameters.ginkgoSkip), 1), eq(parameters.os, 'windows'))) }}
18+
then
19+
SKIP="|LinuxOnly"
20+
elif ${{ lower(eq(parameters.os, 'windows')) }}
21+
then
22+
SKIP="LinuxOnly"
23+
fi
24+
25+
# Taint Linux nodes so that windows tests do not run on them
26+
if ${{ lower(eq(parameters.os, 'windows')) }}
27+
then
28+
kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule
29+
fi
30+
31+
# Depreciating flags. Change once k8s minimum version supported is > 1.24
32+
# nodes -> procs
33+
# flakeAttempts -> flake-attempts
34+
# dryRun -> dry-run
35+
36+
./ginkgo --nodes=${{ parameters.processes }} \
37+
./e2e.test -- \
38+
--num-nodes=2 \
39+
--provider=skeleton \
40+
--ginkgo.focus='${{ parameters.ginkgoFocus }}' \
41+
--ginkgo.skip="${{ parameters.ginkgoSkip }}$SKIP" \
42+
--ginkgo.flakeAttempts=${{ parameters.attempts }} \
43+
--ginkgo.v \
44+
--node-os-distro=${{ parameters.os }} \
45+
--kubeconfig=$HOME/.kube/config
46+
47+
# Untaint Linux nodes once testing is complete
48+
if ${{ lower(eq(parameters.os, 'windows')) }}
49+
then
50+
kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
51+
fi
52+
name: ${{ parameters.name }}
53+
displayName: k8s E2E - ${{ parameters.testName }}

‎.pipelines/npm/npm-cni-integration-test.yaml

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
parameters:
2+
clusterName: ""
3+
os: ""
4+
sub: ""
5+
os_version: ""
6+
tag: ""
7+
8+
jobs:
9+
- job: npm_k8se2e
10+
displayName: "NPM k8s E2E"
11+
dependsOn: ${{ parameters.dependsOn }}
12+
condition: and(succeeded(), ${{ or(contains(parameters.os_version, '2022'), eq(parameters.os, 'linux')) }} )
13+
pool:
14+
name: $(BUILD_POOL_NAME_DEFAULT)
15+
demands:
16+
- agent.os -equals Linux
17+
- Role -equals Build
18+
steps:
19+
- task: AzureCLI@2
20+
displayName: "Deploy NPM to Test Cluster"
21+
inputs:
22+
azureSubscription: ${{ parameters.sub }}
23+
scriptType: "bash"
24+
scriptLocation: "inlineScript"
25+
inlineScript: |
26+
set -ex
27+
28+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
29+
30+
# deploy azure-npm
31+
if ${{ lower(eq(parameters.os, 'windows')) }}
32+
then
33+
# Windows
34+
kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
35+
kubectl set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-ltsc2022-${{ parameters.tag }}
36+
kubectl rollout status -n kube-system daemonset/azure-npm-win
37+
38+
# konnectivity agent tends to fail after rollout. Give it time to recover
39+
sleep 60
40+
# Taint Linux (system) nodes so windows tests do not run on them
41+
kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule
42+
else
43+
# Linux
44+
kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
45+
kubectl set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:${{ parameters.tag }}
46+
kubectl rollout status -n kube-system daemonset/azure-npm
47+
fi
48+
49+
kubectl get po -n kube-system -owide -A
50+
51+
# FQDN=`az aks show -n $CLUSTER_NAME -g $CLUSTER_NAME --query fqdn -o tsv`
52+
FQDN=`az aks show -g ${{ parameters.clusterName }}-$(make revision) -n ${{ parameters.clusterName }}-$(make revision) --query fqdn -o tsv`
53+
echo $FQDN
54+
echo "##vso[task.setvariable variable=FQDN]$FQDN"
55+
56+
- download: current
57+
artifact: Test
58+
59+
- bash: |
60+
# NetworkPolicy between server and...
61+
focus="\
62+
client should enforce policy to allow traffic only from a different namespace, based on NamespaceSelector|\
63+
client should deny egress from pods based on PodSelector|\
64+
client should enforce multiple, stacked policies with overlapping podSelectors|\
65+
client should enforce egress policy allowing traffic to a server in a different namespace based on PodSelector and NamespaceSelector|\
66+
client should work with Ingress, Egress specified together|\
67+
client should enforce ingress policy allowing any port traffic to a server on a specific protocol|\
68+
client should not allow access by TCP when a policy specifies only UDP|\
69+
client should allow egress access to server in CIDR block|\
70+
client should enforce policy based on Ports|\
71+
client should support allow-all policy|\
72+
client should enforce updated policy|\
73+
client should support denying of egress traffic on the client side|\
74+
client should stop enforcing policies after they are deleted|\
75+
client should support a 'default-deny-ingress' policy"
76+
77+
chmod +x $(Pipeline.Workspace)/Test/e2e.test
78+
79+
KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 \
80+
$(Pipeline.Workspace)/Test/e2e.test \
81+
--provider=local \
82+
--ginkgo.focus="$focus" \
83+
--ginkgo.skip="NetworkPolicyLegacy|SCTP" \
84+
--kubeconfig=$HOME/.kube/config
85+
86+
# Untaint Linux (system) nodes once testing is complete
87+
if ${{ lower(eq(parameters.os, 'windows')) }}
88+
then
89+
kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
90+
fi
91+
displayName: "Run Kubernetes e2e.test"
92+
93+

‎.pipelines/pipeline.yaml

+7-47
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ stages:
273273
echo $TAG
274274
echo $CURRENT_VERSION
275275
echo "Checking if branch up to date with master"
276-
276+
277277
- stage: publish
278278
displayName: Publish Multiarch Manifests
279279
dependsOn:
@@ -355,16 +355,6 @@ stages:
355355
clusterName: "swifte2e"
356356
osSku: "Ubuntu"
357357

358-
- template: singletenancy/aks/e2e-job-template.yaml
359-
parameters:
360-
name: "aks_ubuntu_18_04_linux_e2e"
361-
displayName: AKS Ubuntu 18.04
362-
arch: 'amd64'
363-
os: 'linux'
364-
clusterType: linux-cniv1-up
365-
clusterName: 'ubuntu18e2e'
366-
k8sVersion: 1.24.9
367-
368358
- template: singletenancy/aks/e2e-job-template.yaml
369359
parameters:
370360
name: "aks_ubuntu_22_linux_e2e"
@@ -373,18 +363,9 @@ stages:
373363
os: 'linux'
374364
clusterType: linux-cniv1-up
375365
clusterName: 'ubuntu22e2e'
366+
vmSize: Standard_B2s
376367
k8sVersion: 1.25
377-
378-
- template: singletenancy/aks/e2e-job-template.yaml
379-
parameters:
380-
name: "aks_windows_19_03_e2e"
381-
displayName: AKS Windows 1903
382-
arch: amd64
383-
os: windows
384-
clusterType: windows-cniv1-up
385-
clusterName: 'win19e2e'
386-
windowsOsSku: 'Windows2019'
387-
os_version: 'ltsc2019'
368+
scaleup: 100
388369

389370
- template: singletenancy/aks/e2e-job-template.yaml
390371
parameters:
@@ -394,28 +375,10 @@ stages:
394375
os: windows
395376
clusterType: windows-cniv1-up
396377
clusterName: 'win22e2e'
378+
vmSize: Standard_B2ms
397379
windowsOsSku: 'Windows2022'
398380
os_version: 'ltsc2022'
399-
400-
- template: singletenancy/aks-engine/e2e-job-template.yaml
401-
parameters:
402-
name: "ubuntu_18_04_linux_e2e"
403-
displayName: Ubuntu 18.04
404-
pipelineBuildImage: "$(BUILD_IMAGE)"
405-
clusterDefinition: "cniLinux1804.json"
406-
clusterDefinitionCniTypeKey: "azureCNIURLLinux"
407-
clusterDefinitionCniBuildOS: "linux"
408-
clusterDefinitionCniBuildExt: ".tgz"
409-
410-
- template: singletenancy/aks-engine/e2e-job-template.yaml
411-
parameters:
412-
name: "windows_19_03_e2e"
413-
displayName: "Windows 1903"
414-
pipelineBuildImage: "$(BUILD_IMAGE)"
415-
clusterDefinition: "cniWindows1903.json"
416-
clusterDefinitionCniTypeKey: "azureCNIURLWindows"
417-
clusterDefinitionCniBuildOS: "windows"
418-
clusterDefinitionCniBuildExt: ".zip"
381+
scaleup: 100
419382

420383
- stage: validate2
421384
displayName: Validate Tags
@@ -436,18 +399,15 @@ stages:
436399
echo $TAG
437400
echo $CURRENT_VERSION
438401
echo "Checking if branch is up to date with master"
439-
402+
440403
- stage: cleanup
441404
displayName: Cleanup
442405
dependsOn:
443406
- "aks_swift_e2e"
444407
- "cilium_e2e"
445-
- "aks_ubuntu_18_04_linux_e2e"
446-
- "aks_windows_19_03_e2e"
408+
- "cilium_overlay_cilium_e2e"
447409
- "aks_ubuntu_22_linux_e2e"
448410
- "aks_windows_22_e2e"
449-
- "ubuntu_18_04_linux_e2e"
450-
- "windows_19_03_e2e"
451411
jobs:
452412
- job: delete_remote_artifacts
453413
displayName: Delete remote artifacts

‎.pipelines/singletenancy/aks-engine/e2e-job-template.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ parameters:
1010
stages:
1111
- stage: ${{ parameters.name }}
1212
displayName: E2E - ${{ parameters.displayName }}
13-
dependsOn:
13+
dependsOn:
1414
- setup
1515
- publish
1616
jobs:
@@ -19,7 +19,7 @@ stages:
1919
timeoutInMinutes: 120
2020
pool:
2121
name: $(BUILD_POOL_NAME_DEFAULT)
22-
demands:
22+
demands:
2323
- agent.os -equals Linux
2424
- Role -equals Build
2525
container:

0 commit comments

Comments
 (0)