ci:[CNI] ACN PR Pipeline speed improvements

.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

@@ -20,11 +20,11 @@ jobs:
           addSpnToEnvironment: true
           inlineScript: |
             set -e
-            make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
+            make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
 
             # sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx"
             # pulling k8s version from AKS.
-            eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }}-$(make revision) -n ${{ parameters.clusterName }}-$(make revision) --query "currentKubernetesVersion")
+            eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }} -n ${{ parameters.clusterName }} --query "currentKubernetesVersion")
             echo $k8sVersion
             curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz
 
@@ -38,7 +38,6 @@ jobs:
           parameters:
             testName: Datapath
             name: datapath
-            clusterName: ${{ parameters.clusterName }}
             ginkgoFocus: '(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api'
             ginkgoSkip: 'SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6'
             os: ${{ parameters.os }}
@@ -49,7 +48,6 @@ jobs:
           parameters:
             testName: DNS
             name: dns
-            clusterName: ${{ parameters.clusterName }}
             ginkgoFocus: '\[sig-network\].DNS.should'
             ginkgoSkip: 'resolv|256 search'
             os: ${{ parameters.os }}
@@ -60,7 +58,6 @@ jobs:
           parameters:
             testName: Kubectl Portforward
             name: portforward
-            clusterName: ${{ parameters.clusterName }}
             ginkgoFocus: '\[sig-cli\].Kubectl.Port'
             ginkgoSkip: ''
             os: ${{ parameters.os }}
@@ -71,7 +68,6 @@ jobs:
           parameters:
             testName: Service Conformance
             name: service
-            clusterName: ${{ parameters.clusterName }}
             ginkgoFocus: 'Services.*\[Conformance\].*'
             ginkgoSkip: ''
             os: ${{ parameters.os }}
@@ -82,7 +78,6 @@ jobs:
           parameters:
             testName: Host Port
             name: hostport
-            clusterName: ${{ parameters.clusterName }}
             ginkgoFocus: '\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort'
             ginkgoSkip: 'SCTP|exists conflict' # Skip slow 5 minute test
             os: ${{ parameters.os }}
@@ -93,7 +88,6 @@ jobs:
           parameters:
             testName: Hybrid Network
             name: hybrid
-            clusterName: ${{ parameters.clusterName }}
             ginkgoFocus: '\[sig-windows\].Hybrid'
             ginkgoSkip: ''
             os: ${{ parameters.os }}

.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml

@@ -1,7 +1,6 @@
 parameters:
   testName: ""
   name: ""
-  clusterName: ""
   ginkgoFocus: ""
   ginkgoSkip: ""
   os: ""

.pipelines/pipeline.yaml

@@ -32,8 +32,10 @@ stages:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
           - script: |
+              # To use the variables below, you must make the respective stage's dependsOn have - setup or it will not retain context of this stage
               BUILD_NUMBER=$(Build.BuildNumber)
               echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})"
+              echo "##vso[task.setvariable variable=commitID;isOutput=true]$(make revision)"
               echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)"
               echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)"
               echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)"
@@ -313,34 +315,45 @@ stages:
     parameters:
       name: "cilium_e2e"
       displayName: Cilium
-      pipelineBuildImage: "$(BUILD_IMAGE)"
       testDropgz: ""
+      clusterType: cilium-podsubnet-up
       clusterName: "ciliume2e"
+      vmSize: Standard_B2ms
+      k8sVersion: ""
+      dependsOn: 'containerize'
 
   - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
     parameters:
       name: "cilium_overlay_cilium_e2e"
       displayName: Cilium on AKS Overlay
-      pipelineBuildImage: "$(BUILD_IMAGE)"
       testDropgz: ""
+      clusterType: cilium-overlay-up
       clusterName: "overlaye2e"
+      vmSize: Standard_B2ms
+      k8sVersion: ""
+      dependsOn: 'containerize'
 
   - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
     parameters:
       name: "azure_cni_overlay_e2e"
       displayName: Azure CNI Overlay
-      pipelineBuildImage: "$(BUILD_IMAGE)"
       testDropgz: ""
+      clusterType: overlay-byocni-up
       clusterName: "azurecnioverlaye2e"
+      vmSize: Standard_B2ms
+      k8sVersion: ""
+      dependsOn: 'containerize'
 
   - template: singletenancy/aks-swift/e2e-job-template.yaml
     parameters:
       name: "aks_swift_e2e"
       displayName: AKS Swift Ubuntu
-      pipelineBuildImage: "$(BUILD_IMAGE)"
       testDropgz: ""
+      clusterType: swift-byocni-up
       clusterName: "swifte2e"
-      osSku: "Ubuntu"
+      vmSize: Standard_B2s
+      k8sVersion: ""
+      dependsOn: 'containerize'
 
   - template: singletenancy/aks/e2e-job-template.yaml
     parameters:
@@ -353,6 +366,7 @@ stages:
       vmSize: Standard_B2s
       k8sVersion: 1.25
       scaleup: 100
+      dependsOn: 'containerize'
 
   - template: singletenancy/aks/e2e-job-template.yaml
     parameters:
@@ -363,17 +377,69 @@ stages:
       clusterType: windows-cniv1-up
       clusterName: 'win22e2e'
       vmSize: Standard_B2ms
-      windowsOsSku: 'Windows2022'
       os_version: 'ltsc2022'
-      scaleup: 100
+      scaleup: 50
+      dependsOn: 'containerize'
 
   - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml
     parameters:
       name: "dualstackoverlay_e2e"
       displayName: AKS DualStack Overlay
       os: linux
-      testDropgz: true
+      clusterType: dualstack-overlay-byocni-up
       clusterName: "dualstackoverlaye2e"
+      vmSize: Standard_B2s
+      dependsOn: 'containerize'
+      testDropgz: true
+
+  - stage: delete
+    displayName: Delete Clusters
+    condition: always()
+    dependsOn:
+      - setup
+      - "azure_cni_overlay_e2e"
+      - "aks_swift_e2e"
+      - "cilium_e2e"
+      - "cilium_overlay_cilium_e2e"
+      - "aks_ubuntu_22_linux_e2e"
+      - "aks_windows_22_e2e"
+      - "dualstackoverlay_e2e"
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - job: delete
+        displayName: Delete Cluster
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        strategy:
+          matrix:
+            cilium_e2e:
+              name: cilium_e2e
+              clusterName: 'ciliume2e'
+            cilium_overlay_cilium_e2e:
+              name: cilium_overlay_cilium_e2e
+              clusterName: 'overlaye2e'
+            azure_cni_overlay_e2e:
+              name: azure_cni_overlay_e2e
+              clusterName: 'azurecnioverlaye2e'
+            aks_swift_e2e:
+              name: aks_swift_e2e
+              clusterName: 'swifte2e'
+            aks_ubuntu_22_linux_e2e:
+              name: aks_ubuntu_22_linux_e2e
+              clusterName: 'ubuntu22e2e'
+            aks_windows_22_e2e:
+              name: aks_windows_22_e2e
+              clusterName: 'win22e2e'
+            dualstackoverlay_e2e:
+              name: dualstackoverlay_e2e
+              clusterName: 'dualstackoverlaye2e'
+        steps:
+          - template: templates/delete-cluster.yaml
+            parameters:
+              name: $(name)
+              clusterName: $(clusterName)-$(commitID)
+              region: $(REGION_AKS_CLUSTER_TEST)
 
   - stage: validate2
     displayName: Validate Tags
@@ -398,6 +464,7 @@ stages:
   - stage: cleanup
     displayName: Cleanup
     dependsOn:
+      - "azure_cni_overlay_e2e"
       - "aks_swift_e2e"
       - "cilium_e2e"
       - "cilium_overlay_cilium_e2e"

.pipelines/singletenancy/aks-swift/e2e-job-template.yaml

@@ -1,27 +1,53 @@
 parameters:
   name: ""
   displayName: ""
-  pipelineBuildImage: "$(BUILD_IMAGE)"
   testDropgz: ""
+  clusterType: ""
   clusterName: ""
-  osSku: ""
+  vmSize: ""
+  k8sVersion: ""
+  dependsOn: ""
+
 stages:
+  - stage: ${{ parameters.clusterName }}
+    displayName: Create Cluster - ${{ parameters.displayName }}
+    dependsOn:
+      - ${{ parameters.dependsOn }}
+      - setup
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - template: ../../templates/create-cluster.yaml
+        parameters:
+          name: ${{ parameters.name }}
+          displayName: ${{ parameters.displayName }}
+          clusterType: ${{ parameters.clusterType }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
+          k8sVersion: ${{ parameters.k8sVersion }}
+          dependsOn: ${{ parameters.dependsOn }}
+          region: $(REGION_AKS_CLUSTER_TEST)
+
   - stage: ${{ parameters.name }}
     displayName: E2E - ${{ parameters.displayName }}
-    dependsOn: 
+    dependsOn:
     - setup
     - publish
+    - ${{ parameters.clusterName }}
     variables:
       TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
       CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ]
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
     condition: and(succeeded(), eq(variables.TAG, variables.CURRENT_VERSION))
     jobs:
       - job: ${{ parameters.name }}
         displayName: Singletenancy AKS Swift Suite - (${{ parameters.name }})
         timeoutInMinutes: 120
         pool:
           name: $(BUILD_POOL_NAME_DEFAULT)
-          demands: 
+          demands:
           - agent.os -equals Linux
           - Role -equals $(CUSTOM_E2E_ROLE)
         variables:
@@ -33,5 +59,4 @@ stages:
             parameters:
               name: ${{ parameters.name }}
               testDropgz: ${{ parameters.testDropgz }}
-              clusterName: ${{ parameters.clusterName }}
-              osSku: ${{ parameters.osSku }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)

.pipelines/singletenancy/aks-swift/e2e-step-template.yaml

@@ -2,7 +2,6 @@ parameters:
   name: ""
   testDropgz: ""
   clusterName: ""
-  osSku: ""
 
 steps:
   - bash: |
@@ -31,25 +30,17 @@ steps:
       addSpnToEnvironment: true
       inlineScript: |
         set -e
-        mkdir -p ~/.kube/
-        echo "Create AKS cluster"
-        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
-        make -C ./hack/aks byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }}
-        echo "Cluster successfully created"
-    displayName: Create test cluster
-    condition: succeeded()
-
-  - script: |
-      echo "install kubetest2 and gsutils"
-      go get github.com/onsi/ginkgo/ginkgo
-      go get github.com/onsi/gomega/...
-      go install github.com/onsi/ginkgo/ginkgo@latest
-      go install sigs.k8s.io/kubetest2@latest
-      go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
-      go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
-      wget https://storage.googleapis.com/pub/gsutil.tar.gz
-      tar xfz gsutil.tar.gz
-      sudo mv gsutil /usr/local/bin
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+        echo "install kubetest2 and gsutils"
+        go get github.com/onsi/ginkgo/ginkgo
+        go get github.com/onsi/gomega/...
+        go install github.com/onsi/ginkgo/ginkgo@latest
+        go install sigs.k8s.io/kubetest2@latest
+        go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
+        go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
+        wget https://storage.googleapis.com/pub/gsutil.tar.gz
+        tar xfz gsutil.tar.gz
+        sudo mv gsutil /usr/local/bin
     name: "installKubetest"
     displayName: "Set up Conformance Tests"
 
@@ -116,19 +107,3 @@ steps:
     name: "Cleanupartifactdir"
     displayName: "Cleanup artifact dir"
     condition: always()
-
-  - task: AzureCLI@2
-    inputs:
-      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
-        set -e
-        echo "Deleting cluster"
-        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
-        make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision)
-        echo "Cluster and resources down"
-    name: "Cleanupcluster"
-    displayName: "Cleanup cluster"
-    condition: always()