test: add azure cni overlay e2e (#2073)

Azure · Jul 31, 2023 · a7d8c6c · a7d8c6c
1 parent a20fb84
commit a7d8c6c
Show file tree

Hide file tree

Showing 8 changed files with 452 additions and 54 deletions.
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
@@ -329,6 +329,14 @@ stages:
       testDropgz: ""
       clusterName: "overlaye2e"
 
+  - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
+    parameters:
+      name: "azure_cni_overlay_e2e"
+      displayName: Azure CNI Overlay
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      testDropgz: ""
+      clusterName: "azurecnioverlaye2e"
+
   - template: singletenancy/aks-swift/e2e-job-template.yaml
     parameters:
       name: "aks_swift_e2e"

diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
@@ -0,0 +1,32 @@
+parameters:
+  name: ""
+  displayName: ""
+  pipelineBuildImage: "$(BUILD_IMAGE)"
+  testDropgz: ""
+  clusterName: ""
+
+stages:
+  - stage: ${{ parameters.name }}
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn: 
+    - setup
+    - publish
+    jobs:
+      - job: ${{ parameters.name }}
+        displayName: Azure CNI Overlay Test Suite - (${{ parameters.name }})
+        timeoutInMinutes: 120
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands: 
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        variables:
+          GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+        steps:
+          - template: azure-cni-overlay-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              testDropgz: ${{ parameters.testDropgz }}
+              clusterName: ${{ parameters.clusterName }}
diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml
@@ -0,0 +1,140 @@
+parameters:
+  name: ""
+  testDropgz: ""
+  clusterName: ""
+
+steps:
+  - bash: |
+      echo $UID
+      sudo rm -rf $(System.DefaultWorkingDirectory)/*
+    displayName: "Set up OS environment"
+
+  - checkout: self
+
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        mkdir -p ~/.kube/
+        echo "Create AKS Overlay cluster"
+        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST)
+        make -C ./hack/aks overlay-byocni-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms
+        echo "Cluster successfully created"
+    displayName: Create Overlay cluster
+    condition: succeeded()
+
+  - script: |
+      ls -lah
+      pwd
+      echo "installing kubectl"
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+      sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+      kubectl cluster-info
+      kubectl get po -owide -A
+    name: "installKubectl"
+    displayName: "Install Kubectl"
+
+  - script: |
+      echo "install kubetest2 and gsutils"
+      go get github.com/onsi/ginkgo/ginkgo
+      go get github.com/onsi/gomega/...
+      go install github.com/onsi/ginkgo/ginkgo@latest
+      go install sigs.k8s.io/kubetest2@latest
+      go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
+      go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
+      wget https://storage.googleapis.com/pub/gsutil.tar.gz
+      tar xfz gsutil.tar.gz
+      sudo mv gsutil /usr/local/bin
+    name: "installKubetest"
+    displayName: "Set up Conformance Tests"
+
+  - script: |
+      echo "Start Integration Tests on Overlay Cluster"
+      echo "deploy ip-masq-agent for overlay"
+      kubectl apply -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
+      cd test/integration/manifests/ip-masq-agent/
+      kubectl apply -f config-custom.yaml
+      kubectl apply -f config-reconcile.yaml
+      cd ../../../..
+      kubectl get po -owide -A
+      CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version)
+      sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=${DROPGZ} INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
+    retryCountOnTaskFailure: 3
+    name: "integrationTest"
+    displayName: "Run CNS Integration Tests on AKS Overlay"
+
+  - script: |
+      echo "Status of the nodes and pods after the test"
+      kubectl get nodes -o wide
+      kubectl get pods -A -o wide
+      echo "Logs will be available as a build artifact"
+      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
+      echo $ARTIFACT_DIR
+      sudo rm -rf $ARTIFACT_DIR
+      sudo mkdir $ARTIFACT_DIR
+      sudo cp test/integration/logs/* $ARTIFACT_DIR
+    name: "GetLogs"
+    displayName: "Get logs"
+    condition: always()
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: test-output
+      pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
+    condition: always()
+
+  - script: |
+      echo "Run Service Conformance E2E"
+      export PATH=${PATH}:/usr/local/bin/gsutil
+      KUBECONFIG=~/.kube/config kubetest2 noop \
+        --test ginkgo -- \
+        --focus-regex "Services.*\[Conformance\].*"
+    name: "servicesConformance"
+    displayName: "Run Services Conformance Tests"
+
+  - script: |
+      echo "Run wireserver and metadata connectivity Tests"
+      bash test/network/wireserver_metadata_test.sh
+    retryCountOnTaskFailure: 3
+    name: "WireserverMetadataConnectivityTests"
+    displayName: "Run Wireserver and Metadata Connectivity Tests"
+
+  - script: |
+      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
+      echo $ARTIFACT_DIR
+      sudo rm -rf $ARTIFACT_DIR
+      sudo rm -rf test/integration/logs
+    name: "Cleanupartifactdir"
+    displayName: "Cleanup artifact dir"
+    condition: always()
+
+  - task: AzureCLI@2
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        echo "Deleting cluster"
+        make -C ./hack/aks azcfg AZCLI=az
+        make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
+        echo "Cluster and resources down"
+    name: "Cleanupcluster"
+    displayName: "Cleanup cluster"
+    condition: always()
diff --git a/test/integration/manifests/cnsconfig/azurecnioverlayconfigmap.yaml b/test/integration/manifests/cnsconfig/azurecnioverlayconfigmap.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cns-config
+  namespace: kube-system
+data:
+  cns_config.json: |
+    {
+      "TelemetrySettings": {
+          "TelemetryBatchSizeBytes": 16384,
+          "TelemetryBatchIntervalInSecs": 15,
+          "RefreshIntervalInSecs": 15,
+          "DisableAll": false,
+          "HeartBeatIntervalInMins": 30,
+          "DebugMode": false,
+          "SnapshotIntervalInMins": 60
+      },
+      "ManagedSettings": {
+          "PrivateEndpoint": "",
+          "InfrastructureNetworkID": "",
+          "NodeID": "",
+          "NodeSyncIntervalInSeconds": 30
+      },
+      "ChannelMode": "CRD",
+      "InitializeFromCNI": true,
+      "ManageEndpointState": false,
+      "ProgramSNATIPTables" : false,
+      "EnableCNIConflistGeneration": true,
+      "CNIConflistFilepath": "/etc/cni/net.d/15-azure-swift-overlay.conflist",
+      "CNIConflistScenario": "v4overlay"
+    }
diff --git a/test/integration/manifests/ip-masq-agent/config-custom.yaml b/test/integration/manifests/ip-masq-agent/config-custom.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ip-masq-agent-config
+  name: azure-ip-masq-agent-config
   namespace: kube-system
   labels:
     component: ip-masq-agent

diff --git a/test/integration/manifests/ip-masq-agent/config-reconcile.yaml b/test/integration/manifests/ip-masq-agent/config-reconcile.yaml
@@ -1,12 +1,10 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ip-masq-agent-config-reconciled
+  name: azure-ip-masq-agent-config-user
   namespace: kube-system
   labels:
     component: ip-masq-agent
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
 data:
   ip-masq-agent-reconciled: |-
     nonMasqueradeCIDRs:

diff --git a/test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml b/test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml
@@ -1,52 +1,91 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: ip-masq-agent
+  name: azure-ip-masq-agent-user
   namespace: kube-system
-  labels:
-    component: ip-masq-agent
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
-      k8s-app: ip-masq-agent
+      k8s-app: azure-ip-masq-agent-user
   template:
     metadata:
       labels:
-        k8s-app: ip-masq-agent
+        k8s-app: azure-ip-masq-agent-user
     spec:
-      hostNetwork: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.azure.com/cluster
+                operator: Exists
+              - key: type
+                operator: NotIn
+                values:
+                - virtual-kubelet
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      automountServiceAccountToken: false
       containers:
-      - name: ip-masq-agent
-        image: mcr.microsoft.com/aks/ip-masq-agent-v2:v0.1.1
-        imagePullPolicy: Always
+      - args:
+        - --v=2
+        - --resync-interval=60
+        image: mcr.microsoft.com/aks/ip-masq-agent-v2:v0.1.7
+        imagePullPolicy: IfNotPresent
+        name: azure-ip-masq-agent
+        resources:
+          limits:
+            cpu: 500m
+            memory: 250Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
         securityContext:
-          privileged: false
           capabilities:
-            add: ["NET_ADMIN", "NET_RAW"]
-        # Uses projected volumes to merge all data in /etc/config
+            add:
+            - NET_ADMIN
+            - NET_RAW
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
         volumeMounts:
-          - name: ip-masq-agent-volume
-            mountPath: /etc/config
-            readOnly: true
+        - mountPath: /etc/config
+          name: azure-ip-masq-agent-config-volume
+        - mountPath: /run/xtables.lock
+          name: iptableslock
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      - effect: NoSchedule
+        operator: Exists
       volumes:
-      - name: ip-masq-agent-volume
+      - name: azure-ip-masq-agent-config-volume
         projected:
+          defaultMode: 420
           sources:
-            # Note these ConfigMaps must be created in the same namespace as the daemonset
-            - configMap:
-              name: ip-masq-agent-config
-              optional: true
+          - configMap:
               items:
-                - key: ip-masq-agent
-                  path: ip-masq-agent
-                  mode: 444
-            - configMap:
-              name: ip-masq-agent-config-reconciled
+              - key: ip-masq-agent
+                mode: 444
+                path: ip-masq-agent
+              name: azure-ip-masq-agent-config
               optional: true
+          - configMap:
               items:
-                # Avoiding duplicate paths
-                - key: ip-masq-agent-reconciled
-                  path: ip-masq-agent-reconciled
-                  mode: 444
+              - key: ip-masq-agent-reconciled
+                mode: 444
+                path: ip-masq-agent-reconciled
+              name: azure-ip-masq-agent-config-user
+              optional: true
+      - hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+        name: iptableslock