ci: Improve CNI|NPM integration test (#2498)

* ci: increase timeout for CNI|NPM intergration test

* add: retry to generate NPM logs

Author: jpayne3506

.pipelines/npm/npm-cni-integration-test.yaml

@@ -16,6 +16,7 @@ jobs:
     displayName: "NPM k8s E2E"
     dependsOn: ${{ parameters.dependsOn }}
     condition: and( and( not(canceled()), not(failed()) ), ${{ or(contains(parameters.os_version, '2022'), eq(parameters.os, 'linux') ) }} , or( contains(variables.CONTROL_SCENARIO, 'npm') , contains(variables.CONTROL_SCENARIO, 'all') ) )
+    timeoutInMinutes: 180 # This is for testing windows, due to the 3m between the 14 tests -> results in 42m of wasted time
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
       demands:
@@ -120,15 +121,17 @@ jobs:
           --ginkgo.focus="$focus" \
           --ginkgo.skip="NetworkPolicyLegacy|SCTP" \
           --kubeconfig=$HOME/.kube/config
-
-          # Untaint Linux (system) nodes once testing is complete
-          if ${{ lower(eq(parameters.os, 'windows')) }}
-          then
-            kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
-          fi
         displayName: "Run Kubernetes e2e.test"
         continueOnError: ${{ parameters.continueOnError }}
 
+      - ${{ if eq(parameters.os, 'windows') }}:
+        - bash: |
+            # Untaint Linux (system) nodes once testing is complete
+            kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
+
+          displayName: Untaint Linux Nodes
+          condition: always()
+
       - bash: |
           npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
           mkdir -p $npmLogs
@@ -137,9 +140,10 @@ jobs:
           npmPodList=`kubectl get pods -n kube-system | grep npm | awk '{print $1}'`
           # capture all logs
           for npmPod in $npmPodList; do
-              kubectl logs -n kube-system $npmPod > $npmLogs/$npmPod-logs.txt
+            kubectl logs -n kube-system $npmPod > $npmLogs/$npmPod-logs.txt
           done
         displayName: Generate NPM pod logs
+        retryCountOnTaskFailure: 3
         condition: always()
 
       - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)