cilium-cni-load-test.yaml

pr: none
trigger: none

stages:
  - stage: creating_aks_cluster
    displayName: "Create AKS Cluster with Cilium"
    jobs:
      - job: create_aks_cluster_with_cilium
        steps:
          - task: AzureCLI@1
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                set -ex
                make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
                make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision) NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
            name: "CreateAksCluster"
            displayName: "Create AKS Cluster"
  - stage: install_cilium
    dependsOn: creating_aks_cluster
    displayName: "Install Cilium on AKS Overlay"
    jobs:
      - job: deploy_cilium_components
        steps:
          - task: AzureCLI@1
            displayName: "Install Cilium, CNS, and ip-masq-agent"
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                set -ex
                az extension add --name aks-preview
                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                ls -lah
                pwd
                kubectl cluster-info
                kubectl get po -owide -A
                echo "Deploy Azure-CNS"
                kubectl apply -f test/integration/manifests/cilium/cns-write-ovly.yaml
                echo "deploy Cilium ConfigMap"
                kubectl apply -f cilium/configmap.yaml
                kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
                echo "install Cilium onto Overlay Cluster"
                kubectl apply -f test/integration/manifests/cilium/cilium-agent
                kubectl apply -f test/integration/manifests/cilium/cilium-operator
                kubectl get po -owide -A
                echo "deploy ip-masq-agent for overlay"
                kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
                cd test/integration/manifests/ip-masq-agent/
                kubectl create configmap config-custom.yaml
                kubectl create configmap config-reconcile.yaml
                cd ../../../..
                kubectl get po -owide -A
  - stage: pod_deployment
    dependsOn: install_cilium
    displayName: "Pod Deployment"
    jobs:
      - job: deploy_pods
        steps:
          - task: AzureCLI@1
            displayName: "Pod Deployment"
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                set -ex
                az extension add --name aks-preview
                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                cd test/integration/load
                go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=4 -scaleup=2400
  - stage: validate_state
    dependsOn: pod_deployment
    displayName: "Validate State"
    jobs:
      - job: validate_state
        steps:
          - task: AzureCLI@1
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                kubectl get pods -A
                make test-validate-state
            name: "ValidateState"
            displayName: "Validate State"
            retryCountOnTaskFailure: 3
  - stage: restart_nodes
    dependsOn: validate_state
    displayName: "Restart Node"
    jobs:
      - job: restart_nodes
        steps:
          - task: AzureCLI@1
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                echo "Scale up the pods and immediated restart the nodes"
                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
                cd test/integration/load
                echo "Scaling the pods down to 100 per node"
                go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -scaleup=1000 -skip-wait=true
                cd ../../../
                echo "Restarting the nodes"
                vmss_name=$(az vmss list -g MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION) --query "[].name" -o tsv)
                make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision) REGION=$(LOCATION) VMSS_NAME=$vmss_name
                cd test/integration/load
                go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -replicas=1000
            name: "RestartNodes"
            displayName: "Restart Nodes"
          - script: |
              export RESTART_CASE=true
              make test-validate-state
            name: "ValidateState"
            displayName: "Validate State"
            retryCountOnTaskFailure: 3
  - stage: connectivity_tests
    dependsOn: restart_nodes
    displayName: "Connectivity Tests"
    jobs:
      - job: cni_tests
        steps:
          - script: |
              echo "install cilium CLI"
              CILIUM_CLI_VERSION=v0.13.2
              CLI_ARCH=amd64
              curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
              sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
              sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
              rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
            name: "InstallCiliumCli"
            displayName: "Install Cilium CLI"
          - task: AzureCLI@1
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                set -ex
                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
            name: "GetCluster"
            displayName: "Get AKS Cluster"
          - script: |
              kubectl delete ns load-test
              cilium connectivity test
            retryCountOnTaskFailure: 6
            name: "CiliumConnectivityTests"
            displayName: "Run Cilium Connectivity Tests"
  - stage: delete
    displayName: "Delete Resources"
    dependsOn:
      - connectivity_tests
    jobs:
      - job: delete_resources
        steps:
          - task: AzureCLI@1
            inputs:
              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                set -ex
                if [ "$(DELETE_RESOURCES)" ]
                then
                  echo "Deleting Cluster and resource group"
                  make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                  make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
                  make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision)
                  echo "Cluster and resources down"
                else
                  echo "Deletion of resources is False"
                fi
            name: "CleanUpCluster"
            displayName: "Cleanup cluster"
    condition: always()