Refactor dualstack from #2098

Azure · Aug 14, 2023 · e2396eb · e2396eb
1 parent 8aefa96
commit e2396eb
Show file tree

Hide file tree

Showing 12 changed files with 69 additions and 92 deletions.
diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml
@@ -24,7 +24,7 @@ jobs:
 
             # sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx"
             # pulling k8s version from AKS.
-            eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }}-$(make revision) -n ${{ parameters.clusterName }}-$(make revision) --query "currentKubernetesVersion")
+            eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }} -n ${{ parameters.clusterName }} --query "currentKubernetesVersion")
             echo $k8sVersion
             curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz
 

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
@@ -320,7 +320,6 @@ stages:
       clusterName: "ciliume2e"
       vmSize: Standard_B2ms
       k8sVersion: ""
-      windowsOsSku: ""
       dependsOn: 'containerize'
 
   - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
@@ -332,7 +331,6 @@ stages:
       clusterName: "overlaye2e"
       vmSize: Standard_B2ms
       k8sVersion: ""
-      windowsOsSku: ""
       dependsOn: 'containerize'
 
   - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
@@ -344,7 +342,6 @@ stages:
       clusterName: "azurecnioverlaye2e"
       vmSize: Standard_B2ms
       k8sVersion: ""
-      windowsOsSku: ""
       dependsOn: 'containerize'
 
   - template: singletenancy/aks-swift/e2e-job-template.yaml
@@ -356,7 +353,6 @@ stages:
       clusterName: "swifte2e"
       vmSize: Standard_B2s
       k8sVersion: ""
-      windowsOsSku: ""
       dependsOn: 'containerize'
 
   - template: singletenancy/aks/e2e-job-template.yaml
@@ -381,7 +377,6 @@ stages:
       clusterType: windows-cniv1-up
       clusterName: 'win22e2e'
       vmSize: Standard_B2ms
-      windowsOsSku: 'Windows2022'
       os_version: 'ltsc2022'
       scaleup: 50
       dependsOn: 'containerize'
@@ -391,9 +386,11 @@ stages:
       name: "dualstackoverlay_e2e"
       displayName: AKS DualStack Overlay
       os: linux
-      testDropgz: true
+      clusterType: dualstack-overlay-byocni-up
       clusterName: "dualstackoverlaye2e"
-
+      vmSize: Standard_B2s
+      dependsOn: 'containerize'
+      testDropgz: true
 
   - stage: delete
     displayName: Delete Clusters
@@ -406,6 +403,7 @@ stages:
       - "cilium_overlay_cilium_e2e"
       - "aks_ubuntu_22_linux_e2e"
       - "aks_windows_22_e2e"
+      - "dualstackoverlay_e2e"
     variables:
       commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
     jobs:
@@ -433,14 +431,16 @@ stages:
             aks_windows_22_e2e:
               name: aks_windows_22_e2e
               clusterName: 'win22e2e'
+            dualstackoverlay_e2e:
+              name: dualstackoverlay_e2e
+              clusterName: 'dualstackoverlaye2e'
         steps:
           - template: templates/delete-cluster.yaml
             parameters:
               name: $(name)
               clusterName: $(clusterName)-$(commitID)
               region: $(REGION_AKS_CLUSTER_TEST)
 
-
   - stage: validate2
     displayName: Validate Tags
     dependsOn:

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -6,11 +6,8 @@ parameters:
   clusterName: ""
   vmSize: ""
   k8sVersion: ""
-  windowsOsSku: ""
   dependsOn: ""
 
-
-
 stages:
   - stage: ${{ parameters.clusterName }}
     displayName: Create Cluster - ${{ parameters.displayName }}
@@ -30,7 +27,6 @@ stages:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           vmSize: ${{ parameters.vmSize }}
           k8sVersion: ${{ parameters.k8sVersion }}
-          windowsOsSku: ${{ parameters.windowsOsSku }}
           dependsOn: ${{ parameters.dependsOn }}
           region: $(REGION_AKS_CLUSTER_TEST)
 

diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml
@@ -7,12 +7,10 @@ parameters:
   clusterName: ""
   vmSize: ""
   k8sVersion: ""
-  windowsOsSku: ""
   os_version: ""
   scaleup: ""
   dependsOn: ""
 
-
 stages:
   - stage: ${{ parameters.clusterName }}
     displayName: Create Cluster - ${{ parameters.displayName }}
@@ -32,7 +30,6 @@ stages:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           vmSize: ${{ parameters.vmSize }}
           k8sVersion: ${{ parameters.k8sVersion }}
-          windowsOsSku: ${{ parameters.windowsOsSku }}
           dependsOn: ${{ parameters.dependsOn }}
           region: $(REGION_AKS_CLUSTER_TEST)
 
@@ -61,13 +58,9 @@ stages:
           - template: e2e-step-template.yaml
             parameters:
               name: ${{ parameters.name }}
-              clusterType: ${{ parameters.clusterType }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
-              vmSize: ${{ parameters.vmSize }}
               arch: ${{ parameters.arch }}
               os: ${{ parameters.os }}
-              k8sVersion: ${{ parameters.k8sVersion }}
-              windowsOsSku: ${{ parameters.windowsOsSku }}
               os_version: ${{ parameters.os_version }}
               version: $(dropgzVersion)
               cniVersion: $(cniVersion)

diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml
@@ -1,14 +1,8 @@
 parameters:
   name: ""
-  clusterType: ""
   clusterName: ""
-  nodeCount: ""
-  vmSize: ""
-  k8sVersion: ""
-  version: ""
+  arch: ""
   os: ""
-  windowsOsSku: ""
-  cniVersion: ""
   os_version: ""
   scaleup: ""
 

diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
@@ -6,7 +6,6 @@ parameters:
   clusterName: ""
   vmSize: ""
   k8sVersion: ""
-  windowsOsSku: ""
   dependsOn: ""
 
 stages:
@@ -28,7 +27,6 @@ stages:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           vmSize: ${{ parameters.vmSize }}
           k8sVersion: ${{ parameters.k8sVersion }}
-          windowsOsSku: ${{ parameters.windowsOsSku }}
           dependsOn: ${{ parameters.dependsOn }}
           region: $(REGION_AKS_CLUSTER_TEST)
 

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
@@ -6,7 +6,6 @@ parameters:
   clusterName: ""
   vmSize: ""
   k8sVersion: ""
-  windowsOsSku: ""
   dependsOn: ""
 
 stages:
@@ -28,7 +27,6 @@ stages:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           vmSize: ${{ parameters.vmSize }}
           k8sVersion: ${{ parameters.k8sVersion }}
-          windowsOsSku: ${{ parameters.windowsOsSku }}
           dependsOn: ${{ parameters.dependsOn }}
           region: $(REGION_AKS_CLUSTER_TEST)
 

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -94,7 +94,7 @@ steps:
       then
           CNS=$(CNS_VERSION) DROPGZ=$(DROP_GZ_VERSION) && echo "Running nightly"
       else
-          CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version) # These could become out of sync TODO
+          CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version)
       fi
       sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=${DROPGZ} INSTALL_CNS=true INSTALL_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
     retryCountOnTaskFailure: 3

diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml
@@ -6,7 +6,6 @@ parameters:
   clusterName: ""
   vmSize: ""
   k8sVersion: ""
-  windowsOsSku: ""
   dependsOn: ""
 
 stages:
@@ -28,7 +27,6 @@ stages:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           vmSize: ${{ parameters.vmSize }}
           k8sVersion: ${{ parameters.k8sVersion }}
-          windowsOsSku: ${{ parameters.windowsOsSku }}
           dependsOn: ${{ parameters.dependsOn }}
           region: $(REGION_AKS_CLUSTER_TEST)
 

diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml
@@ -2,66 +2,68 @@ parameters:
   name: ""
   displayName: ""
   testDropgz: ""
+  clusterType: ""
   clusterName: ""
+  vmSize: ""
+  k8sVersion: ""
+  dependsOn: ""
 
 stages:
+  - stage: ${{ parameters.clusterName }}
+    displayName: Create Cluster - ${{ parameters.displayName }}
+    dependsOn:
+      - ${{ parameters.dependsOn }}
+      - setup
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - template: ../../templates/create-cluster.yaml
+        parameters:
+          name: ${{ parameters.name }}
+          displayName: ${{ parameters.displayName }}
+          clusterType: ${{ parameters.clusterType }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
+          k8sVersion: ${{ parameters.k8sVersion }}
+          dependsOn: ${{ parameters.dependsOn }}
+          region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement
+
   - stage: ${{ parameters.name }}
     displayName: E2E - ${{ parameters.displayName }}
-    dependsOn: 
+    dependsOn:
     - setup
     - publish
+    - ${{ parameters.clusterName }}
+    variables:
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
     jobs:
       - job: ${{ parameters.name }}
         displayName: DualStack Overlay Test Suite - (${{ parameters.name }})
         pool:
           name: $(BUILD_POOL_NAME_DEFAULT)
-          demands: 
+          demands:
+          - agent.os -equals Linux
           - Role -equals $(CUSTOM_E2E_ROLE)
-        variables:
-          GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
-          GOBIN: "$(GOPATH)/bin" # Go binaries path
-          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
         steps:
           - template: dualstackoverlay-e2e-step-template.yaml
             parameters:
               name: ${{ parameters.name }}
               testDropgz: ${{ parameters.testDropgz }}
-              clusterName: ${{ parameters.clusterName }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:
           sub: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
-          clusterName: ${{ parameters.clusterName }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
           os: ${{ parameters.os }}
           dependsOn: ${{ parameters.name }}
           dualstack: true
           dns: true
           portforward: true
           hostport: true
           service: true
-
-      - job: cleanup
-        displayName: "Cleanup"
-        dependsOn:
-          - ${{ parameters.name }}
-          - "cni_k8se2e"
-          - "dualstackoverlay_e2e"
-        pool:
-          name: $(BUILD_POOL_NAME_DEFAULT)
-        condition: always()
-        steps:
-          - task: AzureCLI@1
-            inputs:
-              azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              scriptType: "bash"
-              addSpnToEnvironment: true
-              inlineScript: |
-                set -e
-                echo "Deleting cluster"
-                make -C ./hack/aks azcfg AZCLI=az
-                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
-                make -C ./hack/aks down AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
-                echo "Cluster and resources down"
-            name: "CleanupDualStackOverlaycluster"
-            displayName: "Cleanup DualStack Overlay Cluster"
diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml
@@ -2,6 +2,7 @@ parameters:
   name: ""
   testDropgz: ""
   clusterName: ""
+  cni: "dualstack"
 
 steps:
   - bash: |
@@ -14,34 +15,23 @@ steps:
       echo '##vso[task.prependpath]$(GOROOT)/bin'
     name: "GoEnv"
     displayName: "Set up the Go environment"
+
   - task: AzureCLI@2
     inputs:
       azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       scriptType: "bash"
       addSpnToEnvironment: true
       inlineScript: |
-        echo "Check az version"
-        az version
-        echo "Install az cli extension preview"
-        az extension add --name aks-preview
-        az extension update --name aks-preview
-        mkdir -p ~/.kube/
-        echo "Create AKS DualStack Overlay cluster"
-        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)
-        make -C ./hack/aks dualstack-overlay-byocni-up AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
-        echo "Dualstack Overlay Cluster is successfully created"
-    displayName: Create DualStackOverlay cluster
-    condition: succeeded()
-
-  - script: |
-      ls -lah
-      pwd
-      echo "installing kubectl"
-      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-      sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-      kubectl cluster-info
-      kubectl get po -owide -A
+        set -e
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+        ls -lah
+        pwd
+        echo "installing kubectl"
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+        sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+        kubectl cluster-info
+        kubectl get po -owide -A
     retryCountOnTaskFailure: 3
     name: "installKubectl"
     displayName: "Install Kubectl"
@@ -68,11 +58,11 @@ steps:
       echo "DualStack Overlay Linux control plane Load test"
       sudo go test -timeout 30m -tags load -run ^TestLoad$ -tags=load
       echo "DualStack Overlay Linux control plane CNS validation test"
-      sudo go test -timeout 30m -tags load -cni dualstack -run ^TestValidateState$ -tags=load
+      make test-validate-state OS=${{ parameters.os }} CNI_TYPE=${{ parameters.cni }}
       cd ../datapath
       echo "Dualstack Overlay Linux datapath IPv6 test"
       sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
       echo "Dualstack Overlay Linux datapath IPv4 test"
       sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration
     name: "DualStack_Overlay_Linux_Tests"
-    displayName: "DualStack Overlay Linux Tests"
+    displayName: "DualStack Overlay Linux Tests"