Skip to content

Commit 9604aea

Browse files
paulyufan2paulyufan2
authored
dualstack overlay linux testcases and windows checker (#2098)
dualstack linux test cases and windows checkers
1 parent a82e39e commit 9604aea

12 files changed

+389
-17
lines changed

‎.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,14 @@ jobs:
9999
os: ${{ parameters.os }}
100100
processes: 8
101101
attempts: 3
102+
- ${{ if eq(parameters.dualstack, true) }}:
103+
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
104+
parameters:
105+
testName: DualStack Test
106+
name: DualStack
107+
clusterName: ${{ parameters.clusterName }}
108+
ginkgoFocus: '\[Feature:IPv6DualStack\]'
109+
ginkgoSkip: 'SCTP|session affinity'
110+
os: ${{ parameters.os }}
111+
processes: 8
112+
attempts: 3

‎.pipelines/pipeline.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,14 @@ stages:
367367
os_version: 'ltsc2022'
368368
scaleup: 100
369369

370+
- template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml
371+
parameters:
372+
name: "dualstackoverlay_e2e"
373+
displayName: AKS DualStack Overlay
374+
os: linux
375+
testDropgz: true
376+
clusterName: "dualstackoverlaye2e"
377+
370378
- stage: validate2
371379
displayName: Validate Tags
372380
dependsOn:
@@ -395,6 +403,7 @@ stages:
395403
- "cilium_overlay_cilium_e2e"
396404
- "aks_ubuntu_22_linux_e2e"
397405
- "aks_windows_22_e2e"
406+
- "dualstackoverlay_e2e"
398407
jobs:
399408
- job: delete_remote_artifacts
400409
displayName: Delete remote artifacts

‎.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
parameters:
2+
name: ""
3+
displayName: ""
4+
testDropgz: ""
5+
clusterName: ""
6+
7+
stages:
8+
- stage: ${{ parameters.name }}
9+
displayName: E2E - ${{ parameters.displayName }}
10+
dependsOn:
11+
- setup
12+
- publish
13+
jobs:
14+
- job: ${{ parameters.name }}
15+
displayName: DualStack Overlay Test Suite - (${{ parameters.name }})
16+
pool:
17+
name: $(BUILD_POOL_NAME_DEFAULT)
18+
demands:
19+
- Role -equals $(CUSTOM_E2E_ROLE)
20+
variables:
21+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
22+
GOBIN: "$(GOPATH)/bin" # Go binaries path
23+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
24+
steps:
25+
- template: dualstackoverlay-e2e-step-template.yaml
26+
parameters:
27+
name: ${{ parameters.name }}
28+
testDropgz: ${{ parameters.testDropgz }}
29+
clusterName: ${{ parameters.clusterName }}
30+
31+
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
32+
parameters:
33+
sub: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
34+
clusterName: ${{ parameters.clusterName }}
35+
os: ${{ parameters.os }}
36+
dependsOn: ${{ parameters.name }}
37+
dualstack: true
38+
dns: true
39+
portforward: true
40+
hostport: true
41+
service: true
42+
43+
- job: cleanup
44+
displayName: "Cleanup"
45+
dependsOn:
46+
- ${{ parameters.name }}
47+
- "cni_k8se2e"
48+
- "dualstackoverlay_e2e"
49+
pool:
50+
name: $(BUILD_POOL_NAME_DEFAULT)
51+
condition: always()
52+
steps:
53+
- task: AzureCLI@1
54+
inputs:
55+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
56+
scriptLocation: "inlineScript"
57+
scriptType: "bash"
58+
addSpnToEnvironment: true
59+
inlineScript: |
60+
set -e
61+
echo "Deleting cluster"
62+
make -C ./hack/aks azcfg AZCLI=az
63+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
64+
make -C ./hack/aks down AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
65+
echo "Cluster and resources down"
66+
name: "CleanupDualStackOverlaycluster"
67+
displayName: "Cleanup DualStack Overlay Cluster"

‎.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
parameters:
2+
name: ""
3+
testDropgz: ""
4+
clusterName: ""
5+
6+
steps:
7+
- bash: |
8+
go version
9+
go env
10+
mkdir -p '$(GOBIN)'
11+
mkdir -p '$(GOPATH)/pkg'
12+
mkdir -p '$(modulePath)'
13+
echo '##vso[task.prependpath]$(GOBIN)'
14+
echo '##vso[task.prependpath]$(GOROOT)/bin'
15+
name: "GoEnv"
16+
displayName: "Set up the Go environment"
17+
- task: AzureCLI@2
18+
inputs:
19+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
20+
scriptLocation: "inlineScript"
21+
scriptType: "bash"
22+
addSpnToEnvironment: true
23+
inlineScript: |
24+
echo "Check az version"
25+
az version
26+
echo "Install az cli extension preview"
27+
az extension add --name aks-preview
28+
az extension update --name aks-preview
29+
mkdir -p ~/.kube/
30+
echo "Create AKS DualStack Overlay cluster"
31+
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)
32+
make -C ./hack/aks dualstack-overlay-byocni-up AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
33+
echo "Dualstack Overlay Cluster is successfully created"
34+
displayName: Create DualStackOverlay cluster
35+
condition: succeeded()
36+
37+
- script: |
38+
ls -lah
39+
pwd
40+
echo "installing kubectl"
41+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
42+
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
43+
kubectl cluster-info
44+
kubectl get po -owide -A
45+
retryCountOnTaskFailure: 3
46+
name: "installKubectl"
47+
displayName: "Install Kubectl"
48+
49+
- script: |
50+
echo "deploy ip-masq-agent for overlay"
51+
kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
52+
cd test/integration/manifests/ip-masq-agent/
53+
kubectl create configmap config-custom.yaml
54+
cd ../../../..
55+
kubectl cluster-info
56+
kubectl get node
57+
CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-test-version)
58+
sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=${DROPGZ} INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
59+
kubectl get po -owide -A
60+
retryCountOnTaskFailure: 3
61+
name: "integrationTest"
62+
displayName: "Run CNS Integration Tests on AKS DualStack Overlay"
63+
64+
- script: |
65+
cd test/integration/load
66+
echo "DualStack Overlay Linux control plane Node properties test"
67+
sudo go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load
68+
echo "DualStack Overlay Linux control plane Load test"
69+
sudo go test -timeout 30m -tags load -run ^TestLoad$ -tags=load
70+
echo "DualStack Overlay Linux control plane CNS validation test"
71+
sudo go test -timeout 30m -tags load -cni dualstack -run ^TestValidateState$ -tags=load
72+
cd ../datapath
73+
echo "Dualstack Overlay Linux datapath IPv6 test"
74+
sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
75+
echo "Dualstack Overlay Linux datapath IPv4 test"
76+
sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration
77+
name: "DualStack_Overlay_Linux_Tests"
78+
displayName: "DualStack Overlay Linux Tests"

‎hack/aks/Makefile

+28
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,34 @@ linux-cniv1-up: rg-up overlay-net-up ## Bring up a Linux CNIv1 cluster
228228

229229
@$(MAKE) set-kubeconf
230230

231+
dualstack-overlay-up: rg-up overlay-net-up ## Brings up an dualstack Overlay cluster with Linux node only
232+
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
233+
--node-count $(NODE_COUNT) \
234+
--node-vm-size $(VM_SIZE) \
235+
--network-plugin azure \
236+
--network-plugin-mode overlay \
237+
--subscription $(SUB) \
238+
--ip-families ipv4,ipv6 \
239+
--kubernetes-version 1.26.3 \
240+
--aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview \
241+
--no-ssh-key \
242+
--yes
243+
@$(MAKE) set-kubeconf
244+
245+
dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Overlay BYO CNI cluster
246+
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
247+
--node-count $(NODE_COUNT) \
248+
--node-vm-size $(VM_SIZE) \
249+
--network-plugin none \
250+
--network-plugin-mode overlay \
251+
--subscription $(SUB) \
252+
--ip-families ipv4,ipv6 \
253+
--kubernetes-version 1.26.3 \
254+
--aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview \
255+
--no-ssh-key \
256+
--yes
257+
@$(MAKE) set-kubeconf
258+
231259
down: ## Delete the cluster
232260
$(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes
233261
@$(MAKE) unset-kubeconf

‎hack/aks/README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ AKS Clusters
2424
byocni-up Alias to swift-byocni-up
2525
cilium-up Alias to swift-cilium-up
2626
up Alias to swift-up
27-
overlay-up Brings up an Overlay AzCNI cluster
27+
overlay-up Bring up an Overlay AzCNI cluster
2828
swift-byocni-up Bring up a SWIFT BYO CNI cluster
2929
swift-cilium-up Bring up a SWIFT Cilium cluster
3030
swift-up Bring up a SWIFT AzCNI cluster
3131
windows-cniv1-up Bring up a Windows AzCNIv1 cluster
32+
dualstack-overlay-byocni-up Bring up an dualstack overlay cluster without CNS and CNI installed
3233
down Delete the cluster
3334
vmss-restart Restart the nodes of the cluster
3435
```

‎test/integration/datapath/datapath_linux_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,13 @@ import (
1010
"testing"
1111
"time"
1212

13-
"github.com/Azure/azure-container-networking/test/integration"
13+
k8s "github.com/Azure/azure-container-networking/test/integration"
1414
"github.com/Azure/azure-container-networking/test/integration/goldpinger"
1515
k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils"
1616
"github.com/Azure/azure-container-networking/test/internal/retry"
1717
"github.com/pkg/errors"
1818

1919
appsv1 "k8s.io/api/apps/v1"
20-
apierrors "k8s.io/apimachinery/pkg/api/errors"
2120
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2221
)
2322

‎test/integration/load/load_test.go

+27
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ var (
2525
scaleDownReplicas = flag.Int("scaledown", 1, "Number of replicas to scale down to")
2626
replicas = flag.Int("replicas", 1, "Number of replicas to scale up/down to")
2727
validateStateFile = flag.Bool("validate-statefile", false, "Validate the state file")
28+
validateDualStack = flag.Bool("validate-dualstack", false, "Validate the dualstack overlay")
2829
skipWait = flag.Bool("skip-wait", false, "Skip waiting for pods to be ready")
2930
restartCase = flag.Bool("restart-case", false, "In restart case, skip if we don't find state file")
3031
namespace = "load-test"
@@ -119,6 +120,10 @@ func TestLoad(t *testing.T) {
119120
if *validateStateFile {
120121
t.Run("Validate state file", TestValidateState)
121122
}
123+
124+
if *validateDualStack {
125+
t.Run("Validate dualstack overlay", TestDualStackProperties)
126+
}
122127
}
123128

124129
// TestValidateState validates the state file based on the os and cni type.
@@ -172,3 +177,25 @@ func TestScaleDeployment(t *testing.T) {
172177
t.Fatal(err)
173178
}
174179
}
180+
181+
func TestDualStackProperties(t *testing.T) {
182+
clientset, err := k8sutils.MustGetClientset()
183+
if err != nil {
184+
t.Fatal(err)
185+
}
186+
config := k8sutils.MustGetRestConfig(t)
187+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
188+
defer cancel()
189+
190+
t.Log("Validating the dualstack node labels")
191+
validator, err := validate.CreateValidator(ctx, clientset, config, namespace, *cniType, *restartCase, *osType)
192+
if err != nil {
193+
t.Fatal(err)
194+
}
195+
196+
// validate dualstack overlay scenarios
197+
err = validator.ValidateDualStackControlPlane(ctx)
198+
if err != nil {
199+
t.Fatal(err)
200+
}
201+
}

‎test/integration/setup_test.go

+22-8
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,15 @@ import (
1818
const (
1919
exitFail = 1
2020

21-
envTestDropgz = "TEST_DROPGZ"
22-
envCNIDropgzVersion = "CNI_DROPGZ_VERSION"
23-
envCNSVersion = "CNS_VERSION"
24-
envInstallCNS = "INSTALL_CNS"
25-
envInstallAzilium = "INSTALL_AZILIUM"
26-
envInstallAzureVnet = "INSTALL_AZURE_VNET"
27-
envInstallOverlay = "INSTALL_OVERLAY"
28-
envInstallAzureCNIOverlay = "INSTALL_AZURE_CNI_OVERLAY"
21+
envTestDropgz = "TEST_DROPGZ"
22+
envCNIDropgzVersion = "CNI_DROPGZ_VERSION"
23+
envCNSVersion = "CNS_VERSION"
24+
envInstallCNS = "INSTALL_CNS"
25+
envInstallAzilium = "INSTALL_AZILIUM"
26+
envInstallAzureVnet = "INSTALL_AZURE_VNET"
27+
envInstallOverlay = "INSTALL_OVERLAY"
28+
envInstallAzureCNIOverlay = "INSTALL_AZURE_CNI_OVERLAY"
29+
envInstallDualStackOverlay = "INSTALL_DUALSTACK_OVERLAY"
2930

3031
// relative cns manifest paths
3132
cnsManifestFolder = "manifests/cns"
@@ -182,6 +183,19 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, l
182183
log.Printf("Env %v not set to true, skipping", envInstallAzureCNIOverlay)
183184
}
184185

186+
if installBool5 := os.Getenv(envInstallDualStackOverlay); installBool5 != "" {
187+
if dualStackOverlayScenario, err := strconv.ParseBool(installBool5); err == nil && dualStackOverlayScenario == true {
188+
log.Printf("Env %v set to true, deploy azure-vnet", envInstallDualStackOverlay)
189+
cns.Spec.Template.Spec.InitContainers[0].Args = []string{"deploy", "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o", "/opt/cni/bin/azure-vnet-ipam", "azure-swift-overlay-dualstack.conflist", "-o", "/etc/cni/net.d/10-azure.conflist"}
190+
}
191+
// setup the CNS swiftconfigmap
192+
if err := k8sutils.MustSetupConfigMap(ctx, clientset, cnsSwiftConfigMapPath); err != nil {
193+
return nil, err
194+
}
195+
} else {
196+
log.Printf("Env %v not set to true, skipping", envInstallDualStackOverlay)
197+
}
198+
185199
cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)
186200

187201
log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image)

0 commit comments

Comments
 (0)