Skip to content

Commit 0ac2938

Browse files
authored
ci:Adding cilium dualstack overlay pipeline steps (#2418)
* adding cilium dualstack overlay pipeline steps * changing step order * remove dropgz step * add CNS instead of azurevnet * add v6 CNS state file test * remove nightly pipeline * removing nightly and adding v6 to cilium linux validate * fix func name * removing unneeded steps * addressing comments * remove 120 timeout * cluster type to dualstack-byocni-nokubeproxy-up * setting to INSTALL_OVERLAY * using make to get azure-ipam nad cns * fixing yaml in cil dual steps * update step * skipping bad step for cilium dualstack * update variable name * CNI type * remove file path * fixing CNI versions * removing log outputs * change step * adding cd in directory before running validate * hostport to false * further update * add cilium to dualstack test name
1 parent e735d6e commit 0ac2938

File tree

6 files changed

+372
-6
lines changed

6 files changed

+372
-6
lines changed

.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

+12-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ jobs:
119119
os: ${{ parameters.os }}
120120
processes: 8
121121
attempts: 3
122-
- ${{ if eq(parameters.dualstack, true) }}:
122+
- ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}:
123123
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
124124
parameters:
125125
testName: DualStack Test
@@ -130,3 +130,14 @@ jobs:
130130
os: ${{ parameters.os }}
131131
processes: 8
132132
attempts: 3
133+
- ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}:
134+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
135+
parameters:
136+
testName: DualStack Test|Cilium
137+
name: DualStack
138+
clusterName: ${{ parameters.clusterName }}
139+
ginkgoFocus: '\[Feature:IPv6DualStack\]'
140+
ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135
141+
os: ${{ parameters.os }}
142+
processes: 8
143+
attempts: 3

.pipelines/pipeline.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,18 @@ stages:
420420
k8sVersion: ""
421421
dependsOn: "test"
422422

423+
# Cilium Dualstack Overlay E2E tests
424+
- template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
425+
parameters:
426+
name: "cilium_dualstackoverlay_e2e"
427+
displayName: Cilium on AKS DualStack Overlay
428+
os: linux
429+
clusterType: dualstack-byocni-nokubeproxy-up
430+
clusterName: "cildsovere2e"
431+
vmSize: Standard_B2ms
432+
k8sVersion: ""
433+
dependsOn: "test"
434+
423435
# Cilium Overlay with hubble E2E tests
424436
- template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml
425437
parameters:
@@ -517,6 +529,7 @@ stages:
517529
- aks_swift_vnetscale_e2e
518530
- aks_windows_22_e2e
519531
- dualstackoverlay_e2e
532+
- cilium_dualstackoverlay_e2e
520533
variables:
521534
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
522535
jobs:
@@ -562,6 +575,9 @@ stages:
562575
dualstackoverlay_e2e:
563576
name: dualstackoverlay_e2e
564577
clusterName: "dsovere2e"
578+
cilium_dualstackoverlay_e2e:
579+
name: cilium_dualstackoverlay_e2e
580+
clusterName: "cildsovere2e"
565581
steps:
566582
- template: templates/delete-cluster.yaml
567583
parameters:
@@ -600,6 +616,7 @@ stages:
600616
- aks_ubuntu_22_linux_e2e
601617
- aks_windows_22_e2e
602618
- dualstackoverlay_e2e
619+
- cilium_dualstackoverlay_e2e
603620
jobs:
604621
- job: delete_remote_artifacts
605622
displayName: Delete remote artifacts
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
parameters:
2+
name: ""
3+
displayName: ""
4+
clusterType: ""
5+
clusterName: ""
6+
vmSize: ""
7+
k8sVersion: ""
8+
dependsOn: ""
9+
10+
stages:
11+
- stage: ${{ parameters.clusterName }}
12+
displayName: Create Cluster - ${{ parameters.displayName }}
13+
dependsOn:
14+
- ${{ parameters.dependsOn }}
15+
- setup
16+
pool:
17+
name: $(BUILD_POOL_NAME_DEFAULT)
18+
variables:
19+
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
20+
jobs:
21+
- template: ../../templates/create-cluster.yaml
22+
parameters:
23+
name: ${{ parameters.name }}
24+
displayName: ${{ parameters.displayName }}
25+
clusterType: ${{ parameters.clusterType }}
26+
clusterName: ${{ parameters.clusterName }}-$(commitID)
27+
vmSize: ${{ parameters.vmSize }}
28+
k8sVersion: ${{ parameters.k8sVersion }}
29+
dependsOn: ${{ parameters.dependsOn }}
30+
region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement
31+
32+
- stage: ${{ parameters.name }}
33+
displayName: E2E - ${{ parameters.displayName }}
34+
dependsOn:
35+
- setup
36+
- publish
37+
- ${{ parameters.clusterName }}
38+
variables:
39+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
40+
GOBIN: "$(GOPATH)/bin" # Go binaries path
41+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
42+
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
43+
pool:
44+
name: $(BUILD_POOL_NAME_DEFAULT)
45+
jobs:
46+
- job: ${{ parameters.name }}
47+
displayName: Cilium Dualstack Overlay Test Suite - (${{ parameters.name }})
48+
pool:
49+
name: $(BUILD_POOL_NAME_DEFAULT)
50+
demands:
51+
- agent.os -equals Linux
52+
- Role -equals $(CUSTOM_E2E_ROLE)
53+
steps:
54+
- template: cilium-dualstackoverlay-e2e-step-template.yaml
55+
parameters:
56+
name: ${{ parameters.name }}
57+
clusterName: ${{ parameters.clusterName }}-$(commitID)
58+
59+
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
60+
parameters:
61+
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
62+
clusterName: ${{ parameters.clusterName }}-$(commitID)
63+
os: ${{ parameters.os }}
64+
cni: cilium
65+
dependsOn: ${{ parameters.name }}
66+
dualstack: true
67+
dns: true
68+
portforward: true
69+
service: true
70+
71+
- job: failedE2ELogs
72+
displayName: "Failure Logs"
73+
dependsOn:
74+
- ${{ parameters.name }}
75+
- cni_${{ parameters.os }}
76+
condition: failed()
77+
steps:
78+
- template: ../../templates/log-template.yaml
79+
parameters:
80+
clusterName: ${{ parameters.clusterName }}-$(commitID)
81+
os: ${{ parameters.os }}
82+
cni: cilium
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
parameters:
2+
name: ""
3+
clusterName: ""
4+
5+
steps:
6+
7+
- bash: |
8+
go version
9+
go env
10+
mkdir -p '$(GOBIN)'
11+
mkdir -p '$(GOPATH)/pkg'
12+
mkdir -p '$(modulePath)'
13+
echo '##vso[task.prependpath]$(GOBIN)'
14+
echo '##vso[task.prependpath]$(GOROOT)/bin'
15+
name: "GoEnv"
16+
displayName: "Set up the Go environment"
17+
18+
- task: KubectlInstaller@0
19+
inputs:
20+
kubectlVersion: latest
21+
22+
- task: AzureCLI@1
23+
inputs:
24+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
25+
scriptLocation: "inlineScript"
26+
scriptType: "bash"
27+
addSpnToEnvironment: true
28+
inlineScript: |
29+
set -e
30+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
31+
ls -lah
32+
pwd
33+
kubectl cluster-info
34+
kubectl get po -owide -A
35+
echo "deploy Cilium ConfigMap"
36+
kubectl apply -f test/integration/manifests/cilium/cilium-config-dualstack.yaml
37+
echo "install Cilium ${CILIUM_VERSION_TAG}"
38+
# Passes Cilium image to daemonset and deployment
39+
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
40+
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
41+
# Use different file directories for nightly and current cilium version
42+
kubectl apply -f test/integration/manifests/cilium/cilium-agent
43+
kubectl apply -f test/integration/manifests/cilium/cilium-operator
44+
kubectl get po -owide -A
45+
name: "installCilium"
46+
displayName: "Install Cilium on AKS Dualstack Overlay"
47+
48+
- script: |
49+
echo "install cilium CLI"
50+
if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then
51+
echo "Cilium Agent Version ${BASH_REMATCH[0]}"
52+
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt)
53+
else
54+
echo "Cilium Agent Version ${CILIUM_VERSION_TAG}"
55+
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
56+
fi
57+
CLI_ARCH=amd64
58+
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
59+
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
60+
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
61+
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
62+
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
63+
cilium status
64+
cilium version
65+
name: "installCiliumCLI"
66+
displayName: "Install Cilium CLI"
67+
68+
- script: |
69+
echo "Start Azilium E2E Tests on Overlay Cluster"
70+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true
71+
retryCountOnTaskFailure: 3
72+
name: "aziliumTest"
73+
displayName: "Run Azilium E2E on AKS Overlay"
74+
75+
- script: |
76+
kubectl get pods -A
77+
echo "Waiting < 2 minutes for cilium to be ready"
78+
# Ensure Cilium is ready Xm\Xs
79+
cilium status --wait --wait-duration 2m
80+
retryCountOnTaskFailure: 3
81+
name: "CiliumStatus"
82+
displayName: "Cilium Status"
83+
84+
- script: |
85+
echo "Run Cilium Connectivity Tests"
86+
cilium status
87+
cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption'
88+
retryCountOnTaskFailure: 3
89+
name: "ciliumConnectivityTests"
90+
displayName: "Run Cilium Connectivity Tests"
91+
92+
- script: |
93+
set -e
94+
kubectl get po -owide -A
95+
cd test/integration/datapath
96+
echo "Dualstack Overlay Linux datapath IPv6 test"
97+
go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
98+
echo "Dualstack Overlay Linux datapath IPv4 test"
99+
go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration
100+
retryCountOnTaskFailure: 3
101+
name: "DualStack_Overlay_Linux_Tests"
102+
displayName: "DualStack Overlay Linux Tests"
103+
104+
- script: |
105+
echo "validate pod IP assignment and check systemd-networkd restart"
106+
kubectl get pod -owide -A
107+
# Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change.
108+
# Saves 17 minutes
109+
kubectl delete deploy -n cilium-test echo-external-node
110+
cd test/integration/load
111+
CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$
112+
echo "delete cilium connectivity test resources and re-validate state"
113+
kubectl delete ns cilium-test
114+
kubectl get pod -owide -A
115+
CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$
116+
name: "validatePods"
117+
displayName: "Validate Pods"
118+
119+
- script: |
120+
echo "Run wireserver and metadata connectivity Tests"
121+
bash test/network/wireserver_metadata_test.sh
122+
retryCountOnTaskFailure: 3
123+
name: "WireserverMetadataConnectivityTests"
124+
displayName: "Run Wireserver and Metadata Connectivity Tests"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
apiVersion: v1
2+
data:
3+
agent-not-ready-taint-key: node.cilium.io/agent-not-ready
4+
arping-refresh-period: 30s
5+
auto-direct-node-routes: "false"
6+
bpf-lb-external-clusterip: "false"
7+
bpf-lb-map-max: "65536"
8+
bpf-lb-mode: snat
9+
bpf-map-dynamic-size-ratio: "0.0025"
10+
bpf-policy-map-max: "16384"
11+
bpf-root: /sys/fs/bpf
12+
cgroup-root: /run/cilium/cgroupv2
13+
cilium-endpoint-gc-interval: 5m0s
14+
cluster-id: "0"
15+
cluster-name: default
16+
debug: "false"
17+
disable-cnp-status-updates: "true"
18+
disable-endpoint-crd: "false"
19+
enable-auto-protect-node-port-range: "true"
20+
enable-bgp-control-plane: "false"
21+
enable-bpf-clock-probe: "true"
22+
enable-endpoint-health-checking: "false"
23+
enable-endpoint-routes: "true"
24+
enable-health-check-nodeport: "true"
25+
enable-health-checking: "true"
26+
enable-host-legacy-routing: "true"
27+
enable-hubble: "false"
28+
enable-ipv4: "true"
29+
enable-ipv4-masquerade: "false"
30+
enable-ipv6: "true"
31+
enable-ipv6-masquerade: "false"
32+
enable-k8s-terminating-endpoint: "true"
33+
enable-l2-neigh-discovery: "true"
34+
enable-l7-proxy: "false"
35+
enable-local-node-route: "false"
36+
enable-local-redirect-policy: "false"
37+
enable-metrics: "true"
38+
enable-policy: default
39+
enable-remote-node-identity: "true"
40+
enable-session-affinity: "true"
41+
enable-svc-source-range-check: "true"
42+
enable-vtep: "false"
43+
enable-well-known-identities: "false"
44+
enable-xt-socket-fallback: "true"
45+
identity-allocation-mode: crd
46+
install-iptables-rules: "true"
47+
install-no-conntrack-iptables-rules: "false"
48+
ipam: delegated-plugin
49+
kube-proxy-replacement: strict
50+
kube-proxy-replacement-healthz-bind-address: "0.0.0.0:10256"
51+
local-router-ipv4: 169.254.23.0
52+
local-router-ipv6: "fe80::"
53+
metrics: +cilium_bpf_map_pressure
54+
monitor-aggregation: medium
55+
monitor-aggregation-flags: all
56+
monitor-aggregation-interval: 5s
57+
node-port-bind-protection: "true"
58+
nodes-gc-interval: 5m0s
59+
operator-api-serve-addr: 127.0.0.1:9234
60+
operator-prometheus-serve-addr: :9963
61+
preallocate-bpf-maps: "false"
62+
procfs: /host/proc
63+
prometheus-serve-addr: :9962
64+
remove-cilium-node-taints: "true"
65+
set-cilium-is-up-condition: "true"
66+
sidecar-istio-proxy-image: cilium/istio_proxy
67+
synchronize-k8s-nodes: "true"
68+
tofqdns-dns-reject-response-code: refused
69+
tofqdns-enable-dns-compression: "true"
70+
tofqdns-endpoint-max-ip-per-hostname: "50"
71+
tofqdns-idle-connection-grace-period: 0s
72+
tofqdns-max-deferred-connection-deletes: "10000"
73+
tofqdns-min-ttl: "3600"
74+
tofqdns-proxy-response-max-delay: 100ms
75+
tunnel: disabled
76+
unmanaged-pod-watcher-interval: "15"
77+
vtep-cidr: ""
78+
vtep-endpoint: ""
79+
vtep-mac: ""
80+
vtep-mask: ""
81+
kind: ConfigMap
82+
metadata:
83+
annotations:
84+
meta.helm.sh/release-name: cilium
85+
meta.helm.sh/release-namespace: kube-system
86+
labels:
87+
app.kubernetes.io/managed-by: Helm
88+
name: cilium-config
89+
namespace: kube-system

0 commit comments

Comments
 (0)