Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: [CNI] Replace the bash scripts for CNI load testing with golang test cases #2003

Merged
merged 1 commit into from
Jun 9, 2023
Merged

CI: [CNI] Replace the bash scripts for CNI load testing with golang test cases #2003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 20 additions & 17 deletions .pipelines/cni/cilium/cilium-cni-load-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ stages:
inlineScript: |
set -ex
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP} NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision) NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
name: "CreateAksCluster"
displayName: "Create AKS Cluster"
- stage: install_cilium
Expand All @@ -35,7 +35,7 @@ stages:
inlineScript: |
set -ex
az extension add --name aks-preview
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
ls -lah
pwd
kubectl cluster-info
Expand Down Expand Up @@ -72,10 +72,9 @@ stages:
inlineScript: |
set -ex
az extension add --name aks-preview
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
kubectl apply -f hack/manifests/pod.yaml
kubectl apply -f hack/manifests/hostprocess.yaml
bash hack/scripts/scale_deployment.sh
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
cd test/integration/load
go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=4 -scaleup=2400
- stage: validate_state
dependsOn: pod_deployment
displayName: "Validate State"
Expand All @@ -89,9 +88,9 @@ stages:
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
kubectl get pods -A
bash hack/scripts/validate_state.sh
make test-validate-state
name: "ValidateState"
displayName: "Validate State"
retryCountOnTaskFailure: 3
Expand All @@ -109,18 +108,22 @@ stages:
addSpnToEnvironment: true
inlineScript: |
echo "Scale up the pods and immediated restart the nodes"
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
cd test/integration/load
echo "Scaling the pods down to 100 per node"
bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -s
go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -scaleup=1000 -skip-wait=true
cd ../../../
echo "Restarting the nodes"
vmss_name=$(az vmss list -g MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION) --query "[].name" -o tsv)
make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP} REGION=$(LOCATION) VMSS_NAME=$vmss_name
bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -c
make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision) REGION=$(LOCATION) VMSS_NAME=$vmss_name
cd test/integration/load
go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -replicas=1000
name: "RestartNodes"
displayName: "Restart Nodes"
- script: |
bash hack/scripts/validate_state.sh -r true
export RESTART_CASE=true
make test-validate-state
name: "ValidateState"
displayName: "Validate State"
retryCountOnTaskFailure: 3
Expand Down Expand Up @@ -148,11 +151,11 @@ stages:
addSpnToEnvironment: true
inlineScript: |
set -ex
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
name: "GetCluster"
displayName: "Get AKS Cluster"
- script: |
kubectl delete deployment container -n default
kubectl delete ns load-test
cilium connectivity test
retryCountOnTaskFailure: 6
name: "CiliumConnectivityTests"
Expand All @@ -175,9 +178,9 @@ stages:
if [ "$(DELETE_RESOURCES)" ]
then
echo "Deleting Cluster and resource group"
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}
make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision)
echo "Cluster and resources down"
else
echo "Deletion of resources is False"
Expand Down
5 changes: 2 additions & 3 deletions .pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,13 +131,12 @@ steps:

- script: |
echo "validate pod IP assignment and check systemd-networkd restart"
kubectl apply -f hack/manifests/hostprocess.yaml
kubectl get pod -owide -A
bash hack/scripts/validate_state.sh
make test-validate-state
echo "delete cilium connectivity test resources and re-validate state"
kubectl delete ns cilium-test
kubectl get pod -owide -A
bash hack/scripts/validate_state.sh
make test-validate-state
name: "validatePods"
displayName: "Validate Pods"

Expand Down
5 changes: 2 additions & 3 deletions .pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,12 @@ steps:

- script: |
echo "validate pod IP assignment and check systemd-networkd restart"
kubectl apply -f hack/manifests/hostprocess.yaml
kubectl get pod -owide -A
bash hack/scripts/validate_state.sh
make test-validate-state
echo "delete cilium connectivity test resources and re-validate state"
kubectl delete ns cilium-test
kubectl get pod -owide -A
bash hack/scripts/validate_state.sh
make test-validate-state
name: "validatePods"
displayName: "Validate Pods"

Expand Down
6 changes: 6 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -709,6 +709,8 @@ workspace: ## Set up the Go workspace.
##@ Test

COVER_PKG ?= .
#Restart case is used for cni load test pipeline for restarting the nodes cluster.
RESTART_CASE ?= false

# COVER_FILTER omits folders with all files tagged with one of 'unit', '!ignore_uncovered', or '!ignore_autogenerated'
test-all: ## run all unit tests.
Expand All @@ -721,6 +723,10 @@ test-integration: ## run all integration tests.
CNS_VERSION=$(CNS_VERSION) \
go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...

test-validate-state:
cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE)
cd ../../..

test-cyclonus: ## run the cyclonus test for npm.
cd test/cyclonus && bash ./test-cyclonus.sh
cd ..
Expand Down
74 changes: 0 additions & 74 deletions hack/scripts/scale_deployment.sh
View file
Open in desktop

This file was deleted.

161 changes: 0 additions & 161 deletions hack/scripts/validate_state.sh
View file
Open in desktop

This file was deleted.

Loading