From 28525f6ba2e15a0f7d0309056fa0c84947fb1910 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 19 Jul 2023 15:30:14 -0700 Subject: [PATCH 01/13] Move Create Cluster --- .pipelines/pipeline.yaml | 56 +++++++++++++++++++ .../aks-swift/e2e-job-template.yaml | 5 +- .../aks-swift/e2e-step-template.yaml | 31 +++++----- .../aks/e2e-cluster-template.yaml | 24 ++++++++ .../singletenancy/aks/e2e-job-template.yaml | 1 + .../singletenancy/aks/e2e-step-template.yaml | 31 +++++----- .../cilium-overlay-e2e-job-template.yaml | 5 +- .../cilium-overlay-e2e-step-template.yaml | 35 ++++++------ .../cilium/cilium-e2e-job-template.yaml | 5 +- .../cilium/cilium-e2e-step-template.yaml | 31 +++++----- 10 files changed, 156 insertions(+), 68 deletions(-) create mode 100644 .pipelines/singletenancy/aks/e2e-cluster-template.yaml diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index bc2c748a27..fea99e3fb8 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -309,6 +309,62 @@ stages: displayName: "Set current tag variable" condition: always() + - stage: cluster + displayName: Create Clusters + dependsOn: + - containerize # Timing off of publish stage. Don't want clusters sitting doing nothing + jobs: + - job: create_cluster + displayName: Create + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + strategy: + matrix: + cilium_e2e: + displayName: Cilium + clusterType: swift-no-kube-proxy-up + clusterName: 'cilium_e2e' + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + cilium_overlay_cilium_e2e: + displayName: Cilium on AKS Overlay + clusterType: overlay-no-kube-proxy-up + clusterName: 'overlaye2e' + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + aks_swift_e2e: + displayName: AKS Swift Ubuntu + clusterType: byocni-up + clusterName: 'swifte2e' + vmSize: Standard_B2 + k8sVersion: "" + windowsOsSku: "" + aks_ubuntu_22_linux_e2e: + displayName: AKS Ubuntu 22 + clusterType: linux-cniv1-up + clusterName: 'ubuntu22e2e' + vmSize: Standard_B2 + k8sVersion: 1.25 + windowsOsSku: "" + aks_windows_22_e2e: + displayName: AKS Windows 2022 + clusterType: windows-cniv1-up + clusterName: 'win22e2e' + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: 'Windows2022' + steps: + - template: singletenancy/aks/e2e-cluster-template.yaml + parameters: + displayName: $(displayName) + clusterType: $(clusterType) + clusterName: $(clusterName) + vmSize: $(vmSize) + k8sVersion: $(k8sVersion) + windowsOsSku: $(windowsOsSku) + - template: singletenancy/cilium/cilium-e2e-job-template.yaml parameters: name: "cilium_e2e" diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index f409ed047c..c9fc66fcd9 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -8,9 +8,10 @@ parameters: stages: - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} - dependsOn: + dependsOn: - setup - publish + - cluster variables: TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] @@ -21,7 +22,7 @@ stages: timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) - demands: + demands: - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) variables: diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 837e6df395..c77df4cb83 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -23,21 +23,21 @@ steps: name: "GoEnv" displayName: "Set up the Go environment" - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - mkdir -p ~/.kube/ - echo "Create AKS cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }} - echo "Cluster successfully created" - displayName: Create test cluster - condition: succeeded() + # - task: AzureCLI@1 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # set -e + # mkdir -p ~/.kube/ + # echo "Create AKS cluster" + # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + # make -C ./hack/aks byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }} + # echo "Cluster successfully created" + # displayName: Create test cluster + # condition: succeeded() - script: | echo "install kubetest2 and gsutils" @@ -54,6 +54,7 @@ steps: displayName: "Set up Conformance Tests" - script: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) ls -lah pwd echo "installing kubectl" diff --git a/.pipelines/singletenancy/aks/e2e-cluster-template.yaml b/.pipelines/singletenancy/aks/e2e-cluster-template.yaml new file mode 100644 index 0000000000..feaa664460 --- /dev/null +++ b/.pipelines/singletenancy/aks/e2e-cluster-template.yaml @@ -0,0 +1,24 @@ +parameters: + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + windowsOsSku: "" + +steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + mkdir -p ~/.kube/ + # echo "Create AKS cluster" + # echo "parameters ${{ parameters.windowsOsSku }}" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" + displayName: Cluster - ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index cbc139bb2e..98ec11786e 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -23,6 +23,7 @@ stages: dependsOn: - setup - publish + - cluster jobs: - job: ${{ parameters.name }} displayName: Singletenancy AKS - (${{ parameters.name }}) diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index 73ae925906..1ec0866de0 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -23,22 +23,23 @@ steps: echo '##vso[task.prependpath]$(GOROOT)/bin' name: "GoEnv" displayName: "Set up the Go environment" - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - mkdir -p ~/.kube/ - echo "Create AKS cluster" - echo "parameters ${{ parameters.windowsOsSku }}" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} - echo "Cluster successfully created" - displayName: Create test cluster + # - task: AzureCLI@1 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # set -e + # mkdir -p ~/.kube/ + # echo "Create AKS cluster" + # echo "parameters ${{ parameters.windowsOsSku }}" + # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + # make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + # echo "Cluster successfully created" + # displayName: Create test cluster - script: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Upload CNI" if [ "${{parameters.os}}" == "windows" ]; then export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index 9029030be3..229b5ba89c 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -8,16 +8,17 @@ parameters: stages: - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} - dependsOn: + dependsOn: - setup - publish + - cluster jobs: - job: ${{ parameters.name }} displayName: Cilium Overlay Test Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) - demands: + demands: - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) variables: diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 4d0e2e046d..f4486d7ae0 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -22,23 +22,24 @@ steps: name: "GoEnv" displayName: "Set up the Go environment" - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - mkdir -p ~/.kube/ - echo "Create AKS Overlay cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) - make -C ./hack/aks overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms - echo "Cluster successfully created" - displayName: Create Overlay cluster - condition: succeeded() + # - task: AzureCLI@1 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # set -e + # mkdir -p ~/.kube/ + # echo "Create AKS Overlay cluster" + # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) + # make -C ./hack/aks overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms + # echo "Cluster successfully created" + # displayName: Create Overlay cluster + # condition: succeeded() - script: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) ls -lah pwd echo "installing kubectl" @@ -172,7 +173,7 @@ steps: displayName: "Validate Pods" - script: | - if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then + if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then kubectl get pod -owide -n cilium-test echo "wait for pod and cilium identity deletion in cilium-test namespace" ns="cilium-test" @@ -191,7 +192,7 @@ steps: echo "##[error]Cilium Identities still present in cilium-test namespace" else printf -- "Identities deleted from cilium-test namespace\n" - fi + fi else echo "skip cilium identities check for PR pipeline" fi diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 293fd49aab..1cc25ba57e 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -8,9 +8,10 @@ parameters: stages: - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} - dependsOn: + dependsOn: - setup - publish + - cluster variables: TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] @@ -21,7 +22,7 @@ stages: timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) - demands: + demands: - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) variables: diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index cb76ae4a98..5bb26d2552 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -22,23 +22,24 @@ steps: name: "GoEnv" displayName: "Set up the Go environment" - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - mkdir -p ~/.kube/ - echo "Create AKS cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms - echo "Cluster successfully created" - displayName: Create test cluster - condition: succeeded() + # - task: AzureCLI@1 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # set -e + # mkdir -p ~/.kube/ + # echo "Create AKS cluster" + # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + # make -C ./hack/aks swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms + # echo "Cluster successfully created" + # displayName: Create test cluster + # condition: succeeded() - script: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) ls -lah pwd echo "installing kubectl" From 05f2c81fa7a5e50242bbc443d1aef4b1f58cbe1f Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 24 Jul 2023 12:04:48 -0700 Subject: [PATCH 02/13] Lower Windows scaleup --- .pipelines/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index fea99e3fb8..c2c8e2c988 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -421,7 +421,7 @@ stages: vmSize: Standard_B2ms windowsOsSku: 'Windows2022' os_version: 'ltsc2022' - scaleup: 100 + scaleup: 50 - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml parameters: From b5eb0a417abeb6521b74d14f5ef836f55ad94127 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 24 Jul 2023 12:06:13 -0700 Subject: [PATCH 03/13] Delete echo-external-node for Cilium validate --- .pipelines/pipeline.yaml | 6 +-- .../aks-swift/e2e-step-template.yaml | 30 ++++++----- .../singletenancy/aks/e2e-step-template.yaml | 38 ++++++++------ .../cilium-overlay-e2e-step-template.yaml | 51 +++++++++++-------- .../cilium/cilium-e2e-step-template.yaml | 45 +++++++++------- 5 files changed, 100 insertions(+), 70 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index c2c8e2c988..a2a499d59d 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -323,7 +323,7 @@ stages: cilium_e2e: displayName: Cilium clusterType: swift-no-kube-proxy-up - clusterName: 'cilium_e2e' + clusterName: 'ciliume2e' vmSize: Standard_B2ms k8sVersion: "" windowsOsSku: "" @@ -338,14 +338,14 @@ stages: displayName: AKS Swift Ubuntu clusterType: byocni-up clusterName: 'swifte2e' - vmSize: Standard_B2 + vmSize: Standard_B2s k8sVersion: "" windowsOsSku: "" aks_ubuntu_22_linux_e2e: displayName: AKS Ubuntu 22 clusterType: linux-cniv1-up clusterName: 'ubuntu22e2e' - vmSize: Standard_B2 + vmSize: Standard_B2s k8sVersion: 1.25 windowsOsSku: "" aks_windows_22_e2e: diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index c77df4cb83..747b4b825a 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -39,17 +39,23 @@ steps: # displayName: Create test cluster # condition: succeeded() - - script: | - echo "install kubetest2 and gsutils" - go get github.com/onsi/ginkgo/ginkgo - go get github.com/onsi/gomega/... - go install github.com/onsi/ginkgo/ginkgo@latest - go install sigs.k8s.io/kubetest2@latest - go install sigs.k8s.io/kubetest2/kubetest2-noop@latest - go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest - wget https://storage.googleapis.com/pub/gsutil.tar.gz - tar xfz gsutil.tar.gz - sudo mv gsutil /usr/local/bin + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo "install kubetest2 and gsutils" + go get github.com/onsi/ginkgo/ginkgo + go get github.com/onsi/gomega/... + go install github.com/onsi/ginkgo/ginkgo@latest + go install sigs.k8s.io/kubetest2@latest + go install sigs.k8s.io/kubetest2/kubetest2-noop@latest + go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest + wget https://storage.googleapis.com/pub/gsutil.tar.gz + tar xfz gsutil.tar.gz + sudo mv gsutil /usr/local/bin name: "installKubetest" displayName: "Set up Conformance Tests" @@ -128,7 +134,7 @@ steps: set -e echo "Deleting cluster" make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster and resources down" name: "Cleanupcluster" displayName: "Cleanup cluster" diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index 1ec0866de0..3e1f1568b4 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -38,22 +38,28 @@ steps: # make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} # echo "Cluster successfully created" # displayName: Create test cluster - - script: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Upload CNI" - if [ "${{parameters.os}}" == "windows" ]; then - export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) - envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - - kubectl rollout status daemonset/azure-cni -n kube-system - echo "Deploying on windows nodes" - export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='windows' ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_DROPGZ_VERSION=${{ parameters.version }}) - envsubst < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f - - kubectl rollout status daemonset/azure-cni-windows -n kube-system - else - export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) - envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - - kubectl rollout status daemonset/azure-cni -n kube-system - fi + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Upload CNI" + if [ "${{parameters.os}}" == "windows" ]; then + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - + kubectl rollout status daemonset/azure-cni -n kube-system + echo "Deploying on windows nodes" + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='windows' ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_DROPGZ_VERSION=${{ parameters.version }}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f - + kubectl rollout status daemonset/azure-cni-windows -n kube-system + else + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - + kubectl rollout status daemonset/azure-cni -n kube-system + fi name: "UploadCni" displayName: "Upload CNI" - task: AzureCLI@1 diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index f4486d7ae0..b82452e2e9 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -38,27 +38,33 @@ steps: # displayName: Create Overlay cluster # condition: succeeded() - - script: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - ls -lah - pwd - echo "installing kubectl" - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - kubectl cluster-info - kubectl get po -owide -A - if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then FILE_PATH=-nightly && echo "Running nightly"; fi - echo "deploy Cilium ConfigMap" - kubectl apply -f cilium/configmap.yaml - kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-config.yaml - echo "install Cilium ${CILIUM_VERSION_TAG}" - # Passes Cilium image to daemonset and deployment - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - - # Use different file directories for nightly and current cilium version - kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-agent - kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-operator - kubectl get po -owide -A + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + ls -lah + pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get po -owide -A + if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then FILE_PATH=-nightly && echo "Running nightly"; fi + echo "deploy Cilium ConfigMap" + kubectl apply -f cilium/configmap.yaml + kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-config.yaml + echo "install Cilium ${CILIUM_VERSION_TAG}" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - + # Use different file directories for nightly and current cilium version + kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-agent + kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-operator + kubectl get po -owide -A name: "installCilium" displayName: "Install Cilium on AKS Overlay" @@ -159,6 +165,9 @@ steps: - script: | echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A + # Deleting echo-external-node deployment until version matches TODO. Upstream TODO has change. - Replace todo with vipul work/PR in upstream + # Saves 17 minutes + kubectl delete deploy -n cilium-test echo-external-node if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then echo "Check cilium identities in cilium-test namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 5bb26d2552..0da0bc71ae 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -38,24 +38,30 @@ steps: # displayName: Create test cluster # condition: succeeded() - - script: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - ls -lah - pwd - echo "installing kubectl" - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - kubectl cluster-info - kubectl get po -owide -A - echo "deploy Cilium ConfigMap" - kubectl apply -f cilium/configmap.yaml - kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml - echo "install Cilium ${CILIUM_VERSION_TAG}" - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - - kubectl apply -f test/integration/manifests/cilium/cilium-agent - kubectl apply -f test/integration/manifests/cilium/cilium-operator - kubectl get po -owide -A + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + ls -lah + pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get po -owide -A + echo "deploy Cilium ConfigMap" + kubectl apply -f cilium/configmap.yaml + kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml + echo "install Cilium ${CILIUM_VERSION_TAG}" + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + kubectl get po -owide -A name: "installCilium" displayName: "Install Cilium" @@ -143,6 +149,9 @@ steps: - script: | echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A + # Deleting echo-external-node deployment until version matches TODO. Upstream TODO has change. - Replace todo with vipul work/PR in upstream + # Saves 17 minutes + kubectl delete deploy -n cilium-test echo-external-node make test-validate-state echo "delete cilium connectivity test resources and re-validate state" kubectl delete ns cilium-test From 9aeb12cd923fe7bfce5880e0afc17b57458cd196 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 24 Jul 2023 19:37:11 -0700 Subject: [PATCH 04/13] Change cluster creation from job to stage --- .pipelines/pipeline.yaml | 127 ++++++++++-------- .../aks-swift/e2e-job-template.yaml | 21 ++- .../aks-swift/e2e-step-template.yaml | 2 +- .../aks/e2e-cluster-template.yaml | 43 +++--- .../singletenancy/aks/e2e-job-template.yaml | 14 +- .../cilium-overlay-e2e-job-template.yaml | 18 ++- .../cilium/cilium-e2e-job-template.yaml | 18 ++- 7 files changed, 167 insertions(+), 76 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index a2a499d59d..e1dc3c979f 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -309,61 +309,61 @@ stages: displayName: "Set current tag variable" condition: always() - - stage: cluster - displayName: Create Clusters - dependsOn: - - containerize # Timing off of publish stage. Don't want clusters sitting doing nothing - jobs: - - job: create_cluster - displayName: Create - pool: - name: "$(BUILD_POOL_NAME_DEFAULT)" - strategy: - matrix: - cilium_e2e: - displayName: Cilium - clusterType: swift-no-kube-proxy-up - clusterName: 'ciliume2e' - vmSize: Standard_B2ms - k8sVersion: "" - windowsOsSku: "" - cilium_overlay_cilium_e2e: - displayName: Cilium on AKS Overlay - clusterType: overlay-no-kube-proxy-up - clusterName: 'overlaye2e' - vmSize: Standard_B2ms - k8sVersion: "" - windowsOsSku: "" - aks_swift_e2e: - displayName: AKS Swift Ubuntu - clusterType: byocni-up - clusterName: 'swifte2e' - vmSize: Standard_B2s - k8sVersion: "" - windowsOsSku: "" - aks_ubuntu_22_linux_e2e: - displayName: AKS Ubuntu 22 - clusterType: linux-cniv1-up - clusterName: 'ubuntu22e2e' - vmSize: Standard_B2s - k8sVersion: 1.25 - windowsOsSku: "" - aks_windows_22_e2e: - displayName: AKS Windows 2022 - clusterType: windows-cniv1-up - clusterName: 'win22e2e' - vmSize: Standard_B2ms - k8sVersion: "" - windowsOsSku: 'Windows2022' - steps: - - template: singletenancy/aks/e2e-cluster-template.yaml - parameters: - displayName: $(displayName) - clusterType: $(clusterType) - clusterName: $(clusterName) - vmSize: $(vmSize) - k8sVersion: $(k8sVersion) - windowsOsSku: $(windowsOsSku) + # - stage: cluster + # displayName: Create Clusters + # dependsOn: + # - containerize # Timing off of publish stage. Don't want clusters sitting doing nothing + # jobs: + # - job: create_cluster + # displayName: Create + # pool: + # name: "$(BUILD_POOL_NAME_DEFAULT)" + # strategy: + # matrix: + # cilium_e2e: + # displayName: Cilium + # clusterType: swift-no-kube-proxy-up + # clusterName: 'ciliume2e' + # vmSize: Standard_B2ms + # k8sVersion: "" + # windowsOsSku: "" + # cilium_overlay_cilium_e2e: + # displayName: Cilium on AKS Overlay + # clusterType: overlay-no-kube-proxy-up + # clusterName: 'overlaye2e' + # vmSize: Standard_B2ms + # k8sVersion: "" + # windowsOsSku: "" + # aks_swift_e2e: + # displayName: AKS Swift Ubuntu + # clusterType: byocni-up + # clusterName: 'swifte2e' + # vmSize: Standard_B2s + # k8sVersion: "" + # windowsOsSku: "" + # aks_ubuntu_22_linux_e2e: + # displayName: AKS Ubuntu 22 + # clusterType: linux-cniv1-up + # clusterName: 'ubuntu22e2e' + # vmSize: Standard_B2s + # k8sVersion: 1.25 + # windowsOsSku: "" + # aks_windows_22_e2e: + # displayName: AKS Windows 2022 + # clusterType: windows-cniv1-up + # clusterName: 'win22e2e' + # vmSize: Standard_B2ms + # k8sVersion: "" + # windowsOsSku: 'Windows2022' + # steps: + # - template: singletenancy/aks/e2e-cluster-template.yaml + # parameters: + # displayName: $(displayName) + # clusterType: $(clusterType) + # clusterName: $(clusterName) + # vmSize: $(vmSize) + # k8sVersion: $(k8sVersion) + # windowsOsSku: $(windowsOsSku) - template: singletenancy/cilium/cilium-e2e-job-template.yaml parameters: @@ -371,7 +371,12 @@ stages: displayName: Cilium pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: swift-no-kube-proxy-up clusterName: "ciliume2e" + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + dependsOn: 'containerize' - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml parameters: @@ -379,7 +384,12 @@ stages: displayName: Cilium on AKS Overlay pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: overlay-no-kube-proxy-up clusterName: "overlaye2e" + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + dependsOn: 'containerize' - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml parameters: @@ -395,8 +405,13 @@ stages: displayName: AKS Swift Ubuntu pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: byocni-up clusterName: "swifte2e" + vmSize: Standard_B2s + k8sVersion: "" + windowsOsSku: "" osSku: "Ubuntu" + dependsOn: 'containerize' - template: singletenancy/aks/e2e-job-template.yaml parameters: @@ -409,6 +424,7 @@ stages: vmSize: Standard_B2s k8sVersion: 1.25 scaleup: 100 + dependsOn: 'containerize' - template: singletenancy/aks/e2e-job-template.yaml parameters: @@ -422,6 +438,7 @@ stages: windowsOsSku: 'Windows2022' os_version: 'ltsc2022' scaleup: 50 + dependsOn: 'containerize' - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index c9fc66fcd9..2148c74a6e 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -3,15 +3,34 @@ parameters: displayName: "" pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: "" clusterName: "" osSku: "" + vmSize: "" + k8sVersion: "" + windowsOsSku: "" + dependsOn: "" + + + stages: + - template: ../aks/e2e-cluster-template.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} dependsOn: - setup - publish - - cluster + - ${{ parameters.clusterName }} variables: TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 747b4b825a..a2966f9b31 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -46,6 +46,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "install kubetest2 and gsutils" go get github.com/onsi/ginkgo/ginkgo go get github.com/onsi/gomega/... @@ -60,7 +61,6 @@ steps: displayName: "Set up Conformance Tests" - script: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) ls -lah pwd echo "installing kubectl" diff --git a/.pipelines/singletenancy/aks/e2e-cluster-template.yaml b/.pipelines/singletenancy/aks/e2e-cluster-template.yaml index feaa664460..8bb9118f4f 100644 --- a/.pipelines/singletenancy/aks/e2e-cluster-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-cluster-template.yaml @@ -1,24 +1,35 @@ parameters: + name: "" displayName: "" clusterType: "" clusterName: "" vmSize: "" k8sVersion: "" windowsOsSku: "" + dependsOn: "" -steps: - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - mkdir -p ~/.kube/ - # echo "Create AKS cluster" - # echo "parameters ${{ parameters.windowsOsSku }}" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} - echo "Cluster successfully created" - displayName: Cluster - ${{ parameters.displayName }} +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: ${{ parameters.dependsOn }} + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }} + displayName: Cluster - ${{ parameters.name }} + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + mkdir -p ~/.kube/ + # echo "Create AKS cluster" + # echo "parameters ${{ parameters.windowsOsSku }}" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" + displayName: Cluster - ${{ parameters.clusterType }} diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index 98ec11786e..7325c12226 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -10,8 +10,20 @@ parameters: windowsOsSku: "" os_version: "" scaleup: "" + dependsOn: "" stages: + - template: e2e-cluster-template.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} variables: @@ -23,7 +35,7 @@ stages: dependsOn: - setup - publish - - cluster + - ${{ parameters.clusterName }} jobs: - job: ${{ parameters.name }} displayName: Singletenancy AKS - (${{ parameters.name }}) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index 229b5ba89c..8506754962 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -3,15 +3,31 @@ parameters: displayName: "" pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: "" clusterName: "" + vmSize: "" + k8sVersion: "" + windowsOsSku: "" + dependsOn: "" stages: + - template: ../aks/e2e-cluster-template.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} dependsOn: - setup - publish - - cluster + - ${{ parameters.clusterName }} jobs: - job: ${{ parameters.name }} displayName: Cilium Overlay Test Suite - (${{ parameters.name }}) diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 1cc25ba57e..8a5a7c32b0 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -3,15 +3,31 @@ parameters: displayName: "" pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: "" clusterName: "" + vmSize: "" + k8sVersion: "" + windowsOsSku: "" + dependsOn: "" stages: + - template: ../aks/e2e-cluster-template.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} dependsOn: - setup - publish - - cluster + - ${{ parameters.clusterName }} variables: TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] From 5142e955d60918b4d8a583d2790175397050dd44 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 25 Jul 2023 10:45:32 -0700 Subject: [PATCH 05/13] Decrease k8utils retry delay time --- test/internal/k8sutils/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/internal/k8sutils/utils.go b/test/internal/k8sutils/utils.go index 7697e43c31..61059130bc 100644 --- a/test/internal/k8sutils/utils.go +++ b/test/internal/k8sutils/utils.go @@ -36,8 +36,8 @@ const ( SubnetNameLabel = "kubernetes.azure.com/podnetwork-subnet" // RetryAttempts is the number of times to retry a test. - RetryAttempts = 30 - RetryDelay = 30 * time.Second + RetryAttempts = 90 + RetryDelay = 10 * time.Second ) var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file") From b0a74198dde8fb157c32f38e1ade761ea196a2dd Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 25 Jul 2023 10:50:59 -0700 Subject: [PATCH 06/13] Adjust submodule pipeline --- .pipelines/submodules-pipeline.yaml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.pipelines/submodules-pipeline.yaml b/.pipelines/submodules-pipeline.yaml index 900a4b81f6..9a93092455 100644 --- a/.pipelines/submodules-pipeline.yaml +++ b/.pipelines/submodules-pipeline.yaml @@ -250,7 +250,12 @@ stages: displayName: Cilium pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true - clusterName: "submodules-ciliume2e" + clusterType: swift-no-kube-proxy-up + clusterName: "submodciliume2e" + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + dependsOn: 'containerize' - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml parameters: @@ -258,7 +263,12 @@ stages: displayName: Cilium on AKS Overlay pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true - clusterName: "submodules-overlaye2e" + clusterType: overlay-no-kube-proxy-up + clusterName: "submodsoverlaye2e" + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + dependsOn: 'containerize' - template: singletenancy/aks-swift/e2e-job-template.yaml parameters: @@ -266,14 +276,20 @@ stages: displayName: AKS Swift pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true + clusterType: byocni-up clusterName: "submodswift" osSku: "Ubuntu" + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + dependsOn: 'containerize' - stage: cleanup displayName: Cleanup dependsOn: - cilium_e2e - aks_swift_e2e + - cilium_overlay_e2e jobs: - job: delete_remote_artifacts displayName: Delete remote artifacts From 76c95f54bc0c23fb23a93d13893eab5026d13ebc Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 25 Jul 2023 13:39:16 -0700 Subject: [PATCH 07/13] Remove test comments --- .pipelines/pipeline.yaml | 56 ------------------- .../aks-swift/e2e-step-template.yaml | 16 ------ .../singletenancy/aks/e2e-step-template.yaml | 16 +----- .../cilium-overlay-e2e-step-template.yaml | 16 ------ .../cilium/cilium-e2e-step-template.yaml | 16 ------ 5 files changed, 1 insertion(+), 119 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index e1dc3c979f..38c152f493 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -309,62 +309,6 @@ stages: displayName: "Set current tag variable" condition: always() - # - stage: cluster - # displayName: Create Clusters - # dependsOn: - # - containerize # Timing off of publish stage. Don't want clusters sitting doing nothing - # jobs: - # - job: create_cluster - # displayName: Create - # pool: - # name: "$(BUILD_POOL_NAME_DEFAULT)" - # strategy: - # matrix: - # cilium_e2e: - # displayName: Cilium - # clusterType: swift-no-kube-proxy-up - # clusterName: 'ciliume2e' - # vmSize: Standard_B2ms - # k8sVersion: "" - # windowsOsSku: "" - # cilium_overlay_cilium_e2e: - # displayName: Cilium on AKS Overlay - # clusterType: overlay-no-kube-proxy-up - # clusterName: 'overlaye2e' - # vmSize: Standard_B2ms - # k8sVersion: "" - # windowsOsSku: "" - # aks_swift_e2e: - # displayName: AKS Swift Ubuntu - # clusterType: byocni-up - # clusterName: 'swifte2e' - # vmSize: Standard_B2s - # k8sVersion: "" - # windowsOsSku: "" - # aks_ubuntu_22_linux_e2e: - # displayName: AKS Ubuntu 22 - # clusterType: linux-cniv1-up - # clusterName: 'ubuntu22e2e' - # vmSize: Standard_B2s - # k8sVersion: 1.25 - # windowsOsSku: "" - # aks_windows_22_e2e: - # displayName: AKS Windows 2022 - # clusterType: windows-cniv1-up - # clusterName: 'win22e2e' - # vmSize: Standard_B2ms - # k8sVersion: "" - # windowsOsSku: 'Windows2022' - # steps: - # - template: singletenancy/aks/e2e-cluster-template.yaml - # parameters: - # displayName: $(displayName) - # clusterType: $(clusterType) - # clusterName: $(clusterName) - # vmSize: $(vmSize) - # k8sVersion: $(k8sVersion) - # windowsOsSku: $(windowsOsSku) - - template: singletenancy/cilium/cilium-e2e-job-template.yaml parameters: name: "cilium_e2e" diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index a2966f9b31..afc271fdf7 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -23,22 +23,6 @@ steps: name: "GoEnv" displayName: "Set up the Go environment" - # - task: AzureCLI@1 - # inputs: - # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - # scriptLocation: "inlineScript" - # scriptType: "bash" - # addSpnToEnvironment: true - # inlineScript: | - # set -e - # mkdir -p ~/.kube/ - # echo "Create AKS cluster" - # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - # make -C ./hack/aks byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }} - # echo "Cluster successfully created" - # displayName: Create test cluster - # condition: succeeded() - - task: AzureCLI@1 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index 3e1f1568b4..f340a8d0ba 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -23,21 +23,7 @@ steps: echo '##vso[task.prependpath]$(GOROOT)/bin' name: "GoEnv" displayName: "Set up the Go environment" - # - task: AzureCLI@1 - # inputs: - # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - # scriptLocation: "inlineScript" - # scriptType: "bash" - # addSpnToEnvironment: true - # inlineScript: | - # set -e - # mkdir -p ~/.kube/ - # echo "Create AKS cluster" - # echo "parameters ${{ parameters.windowsOsSku }}" - # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - # make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} - # echo "Cluster successfully created" - # displayName: Create test cluster + - task: AzureCLI@1 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index b82452e2e9..4421429022 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -22,22 +22,6 @@ steps: name: "GoEnv" displayName: "Set up the Go environment" - # - task: AzureCLI@1 - # inputs: - # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - # scriptLocation: "inlineScript" - # scriptType: "bash" - # addSpnToEnvironment: true - # inlineScript: | - # set -e - # mkdir -p ~/.kube/ - # echo "Create AKS Overlay cluster" - # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) - # make -C ./hack/aks overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms - # echo "Cluster successfully created" - # displayName: Create Overlay cluster - # condition: succeeded() - - task: AzureCLI@1 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 0da0bc71ae..7896406a2d 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -22,22 +22,6 @@ steps: name: "GoEnv" displayName: "Set up the Go environment" - # - task: AzureCLI@1 - # inputs: - # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - # scriptLocation: "inlineScript" - # scriptType: "bash" - # addSpnToEnvironment: true - # inlineScript: | - # set -e - # mkdir -p ~/.kube/ - # echo "Create AKS cluster" - # make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - # make -C ./hack/aks swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms - # echo "Cluster successfully created" - # displayName: Create test cluster - # condition: succeeded() - - task: AzureCLI@1 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) From 05aa7ab3cc9f68fd0be427175f02f58a4c5fae61 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Tue, 25 Jul 2023 17:18:21 -0700 Subject: [PATCH 08/13] Addressing Comments --- .pipelines/pipeline.yaml | 7 ++- .../aks-swift/e2e-job-template.yaml | 2 +- .../aks-swift/e2e-step-template.yaml | 1 - .../singletenancy/aks/e2e-job-template.yaml | 2 +- .../cilium-overlay-e2e-job-template.yaml | 2 +- .../cilium-overlay-e2e-step-template.yaml | 2 +- .../cilium/cilium-e2e-job-template.yaml | 2 +- .../cilium/cilium-e2e-step-template.yaml | 2 +- .pipelines/submodules-pipeline.yaml | 8 ++-- .../create-cluster.yaml} | 2 - hack/aks/Makefile | 47 ++++++++++--------- 11 files changed, 37 insertions(+), 40 deletions(-) rename .pipelines/{singletenancy/aks/e2e-cluster-template.yaml => templates/create-cluster.yaml} (92%) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 38c152f493..2b5a81b4f3 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -315,7 +315,7 @@ stages: displayName: Cilium pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" - clusterType: swift-no-kube-proxy-up + clusterType: cilium-podsubnet-up clusterName: "ciliume2e" vmSize: Standard_B2ms k8sVersion: "" @@ -328,7 +328,7 @@ stages: displayName: Cilium on AKS Overlay pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" - clusterType: overlay-no-kube-proxy-up + clusterType: cilium-overlay-up clusterName: "overlaye2e" vmSize: Standard_B2ms k8sVersion: "" @@ -349,12 +349,11 @@ stages: displayName: AKS Swift Ubuntu pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" - clusterType: byocni-up + clusterType: swift-byocni-up clusterName: "swifte2e" vmSize: Standard_B2s k8sVersion: "" windowsOsSku: "" - osSku: "Ubuntu" dependsOn: 'containerize' - template: singletenancy/aks/e2e-job-template.yaml diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index 2148c74a6e..7bd4f8845b 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -14,7 +14,7 @@ parameters: stages: - - template: ../aks/e2e-cluster-template.yaml + - template: ../../templates/create-cluster.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index afc271fdf7..1e5a24ea06 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -2,7 +2,6 @@ parameters: name: "" testDropgz: "" clusterName: "" - osSku: "" steps: - bash: | diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index 7325c12226..cb135a90a1 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -13,7 +13,7 @@ parameters: dependsOn: "" stages: - - template: e2e-cluster-template.yaml + - template: ../../templates/create-cluster.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index 8506754962..bfd1f0e0a5 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -11,7 +11,7 @@ parameters: dependsOn: "" stages: - - template: ../aks/e2e-cluster-template.yaml + - template: ../../templates/create-cluster.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 4421429022..95b6ed5e53 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -149,7 +149,7 @@ steps: - script: | echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until version matches TODO. Upstream TODO has change. - Replace todo with vipul work/PR in upstream + # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. # Saves 17 minutes kubectl delete deploy -n cilium-test echo-external-node if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 8a5a7c32b0..239b853e78 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -11,7 +11,7 @@ parameters: dependsOn: "" stages: - - template: ../aks/e2e-cluster-template.yaml + - template: ../../templates/create-cluster.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 7896406a2d..6ef8bf5195 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -133,7 +133,7 @@ steps: - script: | echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until version matches TODO. Upstream TODO has change. - Replace todo with vipul work/PR in upstream + # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. # Saves 17 minutes kubectl delete deploy -n cilium-test echo-external-node make test-validate-state diff --git a/.pipelines/submodules-pipeline.yaml b/.pipelines/submodules-pipeline.yaml index 9a93092455..beb7b024ba 100644 --- a/.pipelines/submodules-pipeline.yaml +++ b/.pipelines/submodules-pipeline.yaml @@ -250,7 +250,7 @@ stages: displayName: Cilium pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true - clusterType: swift-no-kube-proxy-up + clusterType: cilium-podsubnet-up clusterName: "submodciliume2e" vmSize: Standard_B2ms k8sVersion: "" @@ -263,8 +263,8 @@ stages: displayName: Cilium on AKS Overlay pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true - clusterType: overlay-no-kube-proxy-up - clusterName: "submodsoverlaye2e" + clusterType: cilium-overlay-up + clusterName: "submodoverlaye2e" vmSize: Standard_B2ms k8sVersion: "" windowsOsSku: "" @@ -276,7 +276,7 @@ stages: displayName: AKS Swift pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true - clusterType: byocni-up + clusterType: swift-byocni-up clusterName: "submodswift" osSku: "Ubuntu" vmSize: Standard_B2ms diff --git a/.pipelines/singletenancy/aks/e2e-cluster-template.yaml b/.pipelines/templates/create-cluster.yaml similarity index 92% rename from .pipelines/singletenancy/aks/e2e-cluster-template.yaml rename to .pipelines/templates/create-cluster.yaml index 8bb9118f4f..2c665b9a37 100644 --- a/.pipelines/singletenancy/aks/e2e-cluster-template.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -27,8 +27,6 @@ stages: inlineScript: | set -e mkdir -p ~/.kube/ - # echo "Create AKS cluster" - # echo "parameters ${{ parameters.windowsOsSku }}" make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} echo "Cluster successfully created" diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 9e62eda075..a5fbd05959 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -99,20 +99,6 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --yes @$(MAKE) set-kubeconf -overlay-no-kube-proxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster without kube-proxy for Cilium - $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ - --node-count $(NODE_COUNT) \ - --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ - --network-plugin none \ - --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ - --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ - --no-ssh-key \ - --kube-proxy-config ./kube-proxy.json \ - --yes - @$(MAKE) set-kubeconf - overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ @@ -153,43 +139,58 @@ swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster --yes @$(MAKE) set-kubeconf -swift-no-kube-proxy-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster without kube-proxy for Cilium + +swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ - --network-plugin none \ + --network-plugin azure \ + --network-dataplane cilium \ + --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/CiliumDataplanePreview \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ --no-ssh-key \ - --os-sku $(OS_SKU) \ - --kube-proxy-config ./kube-proxy.json \ --yes @$(MAKE) set-kubeconf -swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster +swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ --network-plugin azure \ - --network-dataplane cilium \ - --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/CiliumDataplanePreview \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ --no-ssh-key \ --yes @$(MAKE) set-kubeconf -swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster +cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster without kube-proxy for Cilium $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ - --network-plugin azure \ + --network-plugin none \ + --network-plugin-mode overlay \ + --pod-cidr 192.168.0.0/16 \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --no-ssh-key \ + --kube-proxy-config ./kube-proxy.json \ + --yes + @$(MAKE) set-kubeconf + +cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster without kube-proxy for Cilium + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --load-balancer-sku basic \ + --network-plugin none \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ --no-ssh-key \ + --os-sku $(OS_SKU) \ + --kube-proxy-config ./kube-proxy.json \ --yes @$(MAKE) set-kubeconf From ad915ef77e5285212c48b918f4fcc5f837309598 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Fri, 28 Jul 2023 15:04:44 -0700 Subject: [PATCH 09/13] Add delete cluster template --- .../aks-swift/e2e-job-template.yaml | 8 ++++ .../aks-swift/e2e-step-template.yaml | 16 -------- .../singletenancy/aks/e2e-job-template.yaml | 30 ++++----------- .../cilium-overlay-e2e-job-template.yaml | 8 ++++ .../cilium-overlay-e2e-step-template.yaml | 15 -------- .../cilium/cilium-e2e-job-template.yaml | 8 ++++ .../cilium/cilium-e2e-step-template.yaml | 16 -------- .pipelines/templates/delete-cluster.yaml | 37 +++++++++++++++++++ 8 files changed, 68 insertions(+), 70 deletions(-) create mode 100644 .pipelines/templates/delete-cluster.yaml diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index 7bd4f8845b..33bbd9016c 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -55,3 +55,11 @@ stages: testDropgz: ${{ parameters.testDropgz }} clusterName: ${{ parameters.clusterName }} osSku: ${{ parameters.osSku }} + + - template: ../../templates/delete-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + dependsOn: ${{ parameters.name }} diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 1e5a24ea06..1915bb8722 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -106,19 +106,3 @@ steps: name: "Cleanupartifactdir" displayName: "Cleanup artifact dir" condition: always() - - - task: AzureCLI@2 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - name: "Cleanupcluster" - displayName: "Cleanup cluster" - condition: always() diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index cb135a90a1..d8c919979c 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -73,28 +73,12 @@ stages: hostport: true dependsOn: ${{ parameters.name }} - - job: cleanup - displayName: "Cleanup" - dependsOn: - - ${{ parameters.name }} - - "cni_k8se2e" - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - condition: always() - steps: - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - displayName: "Delete test cluster" + - template: ../../templates/delete-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + dependsOn: ${{ parameters.name }} diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index bfd1f0e0a5..b86ac22480 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -47,3 +47,11 @@ stages: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} clusterName: ${{ parameters.clusterName }} + + - template: ../../templates/delete-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + dependsOn: ${{ parameters.name }} diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 95b6ed5e53..ee2e521f6b 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -221,18 +221,3 @@ steps: displayName: "Cleanup artifact dir" condition: always() - - task: AzureCLI@2 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az - make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - name: "Cleanupcluster" - displayName: "Cleanup cluster" - condition: always() diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 239b853e78..7667779d10 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -51,3 +51,11 @@ stages: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} clusterName: ${{ parameters.clusterName }} + + - template: ../../templates/delete-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + dependsOn: ${{ parameters.name }} diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 6ef8bf5195..6719ebe25e 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -172,19 +172,3 @@ steps: name: "Cleanupartifactdir" displayName: "Cleanup artifact dir" condition: always() - - - task: AzureCLI@2 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - name: "Cleanupcluster" - displayName: "Cleanup cluster" - condition: always() diff --git a/.pipelines/templates/delete-cluster.yaml b/.pipelines/templates/delete-cluster.yaml new file mode 100644 index 0000000000..17a09aad31 --- /dev/null +++ b/.pipelines/templates/delete-cluster.yaml @@ -0,0 +1,37 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + dependsOn: "" + +stages: +- stage: delete${{ parameters.clusterName }} + condition: always() + displayName: Delete Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} # Stage prior to delete + - ${{ parameters.clusterName }} # Requires matching create cluster stage defined by clusterName + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: delete${{ parameters.name }} + displayName: Delete - ${{ parameters.name }} + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + condition: always() + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + echo "Deleting cluster" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + displayName: Cluster - ${{ parameters.clusterType }} From 254e6625f37934fe2ace8ca910a1a84f218fdc08 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 31 Jul 2023 10:58:30 -0700 Subject: [PATCH 10/13] Single Stage Delete --- .pipelines/pipeline.yaml | 39 +++++++++++++++ .../aks-swift/e2e-job-template.yaml | 8 --- .../singletenancy/aks/e2e-job-template.yaml | 10 ---- .../cilium-overlay-e2e-job-template.yaml | 8 --- .../cilium/cilium-e2e-job-template.yaml | 8 --- .pipelines/submodules-pipeline.yaml | 35 +++++++++++++ .pipelines/templates/delete-cluster.yaml | 49 ++++++------------- 7 files changed, 90 insertions(+), 67 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 2b5a81b4f3..2db99d3d24 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -391,6 +391,45 @@ stages: testDropgz: true clusterName: "dualstackoverlaye2e" + + - stage: delete + displayName: Delete Clusters + condition: always() + dependsOn: + - "aks_swift_e2e" + - "cilium_e2e" + - "cilium_overlay_cilium_e2e" + - "aks_ubuntu_22_linux_e2e" + - "aks_windows_22_e2e" + jobs: + - job: delete + displayName: Delete Cluster + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + strategy: + matrix: + cilium_e2e: + name: cilium_e2e + clusterName: 'ciliume2e' + cilium_overlay_cilium_e2e: + name: cilium_overlay_cilium_e2e + clusterName: 'overlaye2e' + aks_swift_e2e: + name: aks_swift_e2e + clusterName: 'swifte2e' + aks_ubuntu_22_linux_e2e: + name: aks_ubuntu_22_linux_e2e + clusterName: 'ubuntu22e2e' + aks_windows_22_e2e: + name: aks_windows_22_e2e + clusterName: 'win22e2e' + steps: + - template: templates/delete-cluster.yaml + parameters: + name: $(name) + clusterName: $(clusterName) + + - stage: validate2 displayName: Validate Tags dependsOn: diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index 33bbd9016c..7bd4f8845b 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -55,11 +55,3 @@ stages: testDropgz: ${{ parameters.testDropgz }} clusterName: ${{ parameters.clusterName }} osSku: ${{ parameters.osSku }} - - - template: ../../templates/delete-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - dependsOn: ${{ parameters.name }} diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index d8c919979c..18057c52dc 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -72,13 +72,3 @@ stages: service: true hostport: true dependsOn: ${{ parameters.name }} - - - template: ../../templates/delete-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - dependsOn: ${{ parameters.name }} - - diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index b86ac22480..bfd1f0e0a5 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -47,11 +47,3 @@ stages: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} clusterName: ${{ parameters.clusterName }} - - - template: ../../templates/delete-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - dependsOn: ${{ parameters.name }} diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 7667779d10..239b853e78 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -51,11 +51,3 @@ stages: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} clusterName: ${{ parameters.clusterName }} - - - template: ../../templates/delete-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - dependsOn: ${{ parameters.name }} diff --git a/.pipelines/submodules-pipeline.yaml b/.pipelines/submodules-pipeline.yaml index beb7b024ba..72a26856cd 100644 --- a/.pipelines/submodules-pipeline.yaml +++ b/.pipelines/submodules-pipeline.yaml @@ -284,6 +284,41 @@ stages: windowsOsSku: "" dependsOn: 'containerize' + - stage: delete + displayName: Delete Clusters + condition: always() + dependsOn: + - "aks_swift_e2e" + - "cilium_e2e" + - "cilium_overlay_cilium_e2e" + jobs: + - job: delete + displayName: Delete Cluster + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + strategy: + matrix: + cilium_e2e: + name: cilium_e2e + clusterName: 'ciliume2e' + cilium_overlay_cilium_e2e: + name: cilium_overlay_cilium_e2e + clusterName: 'overlaye2e' + aks_swift_e2e: + name: aks_swift_e2e + clusterName: 'swifte2e' + aks_ubuntu_22_linux_e2e: + name: aks_ubuntu_22_linux_e2e + clusterName: 'ubuntu22e2e' + aks_windows_22_e2e: + name: aks_windows_22_e2e + clusterName: 'win22e2e' + steps: + - template: templates/delete-cluster.yaml + parameters: + name: $(name) + clusterName: $(clusterName) + - stage: cleanup displayName: Cleanup dependsOn: diff --git a/.pipelines/templates/delete-cluster.yaml b/.pipelines/templates/delete-cluster.yaml index 17a09aad31..a68c5d7ea4 100644 --- a/.pipelines/templates/delete-cluster.yaml +++ b/.pipelines/templates/delete-cluster.yaml @@ -1,37 +1,20 @@ parameters: name: "" - displayName: "" - clusterType: "" clusterName: "" - dependsOn: "" -stages: -- stage: delete${{ parameters.clusterName }} - condition: always() - displayName: Delete Cluster - ${{ parameters.displayName }} - dependsOn: - - ${{ parameters.dependsOn }} # Stage prior to delete - - ${{ parameters.clusterName }} # Requires matching create cluster stage defined by clusterName - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - jobs: - - job: delete${{ parameters.name }} - displayName: Delete - ${{ parameters.name }} - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - condition: always() - steps: - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - displayName: Cluster - ${{ parameters.clusterType }} +steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + echo "Deleting cluster" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + name: delete + displayName: Delete - ${{ parameters.name }} From 26b552a0923fbb1b72a9510e1ef73cea56e93f26 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Mon, 31 Jul 2023 17:11:59 -0700 Subject: [PATCH 11/13] ACN merge commitID fix --- .../cni/k8s-e2e/k8s-e2e-job-template.yaml | 8 +--- .../cni/k8s-e2e/k8s-e2e-step-template.yaml | 1 - .pipelines/pipeline.yaml | 27 +++++++++--- .../aks-swift/e2e-job-template.yaml | 37 ++++++++++------ .../aks-swift/e2e-step-template.yaml | 3 +- .../singletenancy/aks/e2e-job-template.yaml | 37 ++++++++++------ .../singletenancy/aks/e2e-step-template.yaml | 6 +-- .../azure-cni-overlay-e2e-job-template.yaml | 37 ++++++++++++++-- .../azure-cni-overlay-e2e-step-template.yaml | 40 ++++------------- .../cilium-overlay-e2e-job-template.yaml | 35 +++++++++------ .../cilium-overlay-e2e-step-template.yaml | 5 ++- .../cilium/cilium-e2e-job-template.yaml | 35 +++++++++------ .../cilium/cilium-e2e-step-template.yaml | 3 +- .pipelines/submodules-pipeline.yaml | 20 ++++----- .pipelines/templates/create-cluster.yaml | 43 ++++++++----------- .pipelines/templates/delete-cluster.yaml | 8 ++-- 16 files changed, 200 insertions(+), 145 deletions(-) diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml index 4e5d9c9e58..9698dd4da6 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml @@ -20,7 +20,7 @@ jobs: addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} # sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx" # pulling k8s version from AKS. @@ -38,7 +38,6 @@ jobs: parameters: testName: Datapath name: datapath - clusterName: ${{ parameters.clusterName }} ginkgoFocus: '(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api' ginkgoSkip: 'SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6' os: ${{ parameters.os }} @@ -49,7 +48,6 @@ jobs: parameters: testName: DNS name: dns - clusterName: ${{ parameters.clusterName }} ginkgoFocus: '\[sig-network\].DNS.should' ginkgoSkip: 'resolv|256 search' os: ${{ parameters.os }} @@ -60,7 +58,6 @@ jobs: parameters: testName: Kubectl Portforward name: portforward - clusterName: ${{ parameters.clusterName }} ginkgoFocus: '\[sig-cli\].Kubectl.Port' ginkgoSkip: '' os: ${{ parameters.os }} @@ -71,7 +68,6 @@ jobs: parameters: testName: Service Conformance name: service - clusterName: ${{ parameters.clusterName }} ginkgoFocus: 'Services.*\[Conformance\].*' ginkgoSkip: '' os: ${{ parameters.os }} @@ -82,7 +78,6 @@ jobs: parameters: testName: Host Port name: hostport - clusterName: ${{ parameters.clusterName }} ginkgoFocus: '\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort' ginkgoSkip: 'SCTP|exists conflict' # Skip slow 5 minute test os: ${{ parameters.os }} @@ -93,7 +88,6 @@ jobs: parameters: testName: Hybrid Network name: hybrid - clusterName: ${{ parameters.clusterName }} ginkgoFocus: '\[sig-windows\].Hybrid' ginkgoSkip: '' os: ${{ parameters.os }} diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml index ae4606f243..6ea1d9217a 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml @@ -1,7 +1,6 @@ parameters: testName: "" name: "" - clusterName: "" ginkgoFocus: "" ginkgoSkip: "" os: "" diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 2db99d3d24..88db771282 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -34,6 +34,7 @@ stages: - script: | BUILD_NUMBER=$(Build.BuildNumber) echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" + echo "##vso[task.setvariable variable=commitID;isOutput=true]$(make revision)" echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)" echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" @@ -313,7 +314,6 @@ stages: parameters: name: "cilium_e2e" displayName: Cilium - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" clusterType: cilium-podsubnet-up clusterName: "ciliume2e" @@ -326,7 +326,6 @@ stages: parameters: name: "cilium_overlay_cilium_e2e" displayName: Cilium on AKS Overlay - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" clusterType: cilium-overlay-up clusterName: "overlaye2e" @@ -339,15 +338,18 @@ stages: parameters: name: "azure_cni_overlay_e2e" displayName: Azure CNI Overlay - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: overlay-byocni-up clusterName: "azurecnioverlaye2e" + vmSize: Standard_B2ms + k8sVersion: "" + windowsOsSku: "" + dependsOn: 'containerize' - template: singletenancy/aks-swift/e2e-job-template.yaml parameters: name: "aks_swift_e2e" displayName: AKS Swift Ubuntu - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" clusterType: swift-byocni-up clusterName: "swifte2e" @@ -396,11 +398,15 @@ stages: displayName: Delete Clusters condition: always() dependsOn: + - setup + - "azure_cni_overlay_e2e" - "aks_swift_e2e" - "cilium_e2e" - "cilium_overlay_cilium_e2e" - "aks_ubuntu_22_linux_e2e" - "aks_windows_22_e2e" + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: - job: delete displayName: Delete Cluster @@ -411,23 +417,33 @@ stages: cilium_e2e: name: cilium_e2e clusterName: 'ciliume2e' + region: $(REGION_AKS_CLUSTER_TEST) cilium_overlay_cilium_e2e: name: cilium_overlay_cilium_e2e clusterName: 'overlaye2e' + region: $(REGION_OVERLAY_CLUSTER_TEST) + azure_cni_overlay_e2e: + name: azure_cni_overlay_e2e + clusterName: 'azurecnioverlaye2e' + region: $(REGION_OVERLAY_CLUSTER_TEST) aks_swift_e2e: name: aks_swift_e2e clusterName: 'swifte2e' + region: $(REGION_AKS_CLUSTER_TEST) aks_ubuntu_22_linux_e2e: name: aks_ubuntu_22_linux_e2e clusterName: 'ubuntu22e2e' + region: $(REGION_AKS_CLUSTER_TEST) aks_windows_22_e2e: name: aks_windows_22_e2e clusterName: 'win22e2e' + region: $(REGION_AKS_CLUSTER_TEST) steps: - template: templates/delete-cluster.yaml parameters: name: $(name) - clusterName: $(clusterName) + clusterName: $(clusterName)-$(commitID) + region: $(region) - stage: validate2 @@ -453,6 +469,7 @@ stages: - stage: cleanup displayName: Cleanup dependsOn: + - "azure_cni_overlay_e2e" - "aks_swift_e2e" - "cilium_e2e" - "cilium_overlay_cilium_e2e" diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index 7bd4f8845b..adab194e2e 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -1,11 +1,9 @@ parameters: name: "" displayName: "" - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" clusterType: "" clusterName: "" - osSku: "" vmSize: "" k8sVersion: "" windowsOsSku: "" @@ -14,16 +12,27 @@ parameters: stages: - - template: ../../templates/create-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - vmSize: ${{ parameters.vmSize }} - k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} - dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_AKS_CLUSTER_TEST) - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} @@ -34,6 +43,7 @@ stages: variables: TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] condition: and(succeeded(), eq(variables.TAG, variables.CURRENT_VERSION)) jobs: - job: ${{ parameters.name }} @@ -53,5 +63,4 @@ stages: parameters: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} - clusterName: ${{ parameters.clusterName }} - osSku: ${{ parameters.osSku }} + clusterName: ${{ parameters.clusterName }}-$(commitID) diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 1915bb8722..1ae1dbbd6a 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -29,7 +29,8 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} echo "install kubetest2 and gsutils" go get github.com/onsi/ginkgo/ginkgo go get github.com/onsi/gomega/... diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index 18057c52dc..b2e624d406 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -12,17 +12,29 @@ parameters: scaleup: "" dependsOn: "" + stages: - - template: ../../templates/create-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - vmSize: ${{ parameters.vmSize }} - k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} - dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_AKS_CLUSTER_TEST) - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} @@ -32,6 +44,7 @@ stages: modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" dropgzVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.dropgzVersion'] ] cniVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ] + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] dependsOn: - setup - publish @@ -49,7 +62,7 @@ stages: parameters: name: ${{ parameters.name }} clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} arch: ${{ parameters.arch }} os: ${{ parameters.os }} @@ -63,7 +76,7 @@ stages: - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} datapath: true dns: true diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index f340a8d0ba..f9f43069d8 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -23,7 +23,6 @@ steps: echo '##vso[task.prependpath]$(GOROOT)/bin' name: "GoEnv" displayName: "Set up the Go environment" - - task: AzureCLI@1 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) @@ -31,7 +30,8 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} echo "Upload CNI" if [ "${{parameters.os}}" == "windows" ]; then export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) @@ -56,7 +56,7 @@ steps: addSpnToEnvironment: true inlineScript: | set -e - clusterName=${{ parameters.clusterName }}-$(make revision) + clusterName=${{ parameters.clusterName }} echo "Restarting nodes" for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml index 72a46ea952..fe38a1f2ae 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -1,23 +1,52 @@ parameters: name: "" displayName: "" - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" + clusterType: "" clusterName: "" + vmSize: "" + k8sVersion: "" + windowsOsSku: "" + dependsOn: "" stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_OVERLAY_CLUSTER_TEST) + - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} - dependsOn: + dependsOn: - setup - publish + - ${{ parameters.clusterName }} + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: - job: ${{ parameters.name }} displayName: Azure CNI Overlay Test Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) - demands: + demands: - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) variables: @@ -29,4 +58,4 @@ stages: parameters: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 7a4e0ce536..5a137a127a 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -30,22 +30,14 @@ steps: addSpnToEnvironment: true inlineScript: | set -e - mkdir -p ~/.kube/ - echo "Create AKS Overlay cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) - make -C ./hack/aks overlay-byocni-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms - echo "Cluster successfully created" - displayName: Create Overlay cluster - condition: succeeded() - - - script: | - ls -lah - pwd - echo "installing kubectl" - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - kubectl cluster-info - kubectl get po -owide -A + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get po -owide -A name: "installKubectl" displayName: "Install Kubectl" @@ -122,19 +114,3 @@ steps: name: "Cleanupartifactdir" displayName: "Cleanup artifact dir" condition: always() - - - task: AzureCLI@2 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az - make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - name: "Cleanupcluster" - displayName: "Cleanup cluster" - condition: always() diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index bfd1f0e0a5..5f007f8961 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -1,7 +1,6 @@ parameters: name: "" displayName: "" - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" clusterType: "" clusterName: "" @@ -11,16 +10,27 @@ parameters: dependsOn: "" stages: - - template: ../../templates/create-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - vmSize: ${{ parameters.vmSize }} - k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} - dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_OVERLAY_CLUSTER_TEST) - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} @@ -41,9 +51,10 @@ stages: GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] steps: - template: cilium-overlay-e2e-step-template.yaml parameters: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index ee2e521f6b..4e5caa1da9 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -29,7 +29,8 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} ls -lah pwd echo "installing kubectl" @@ -93,7 +94,7 @@ steps: then CNS=$(CNS_VERSION) DROPGZ=$(DROP_GZ_VERSION) && echo "Running nightly" else - CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version) + CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version) # These could become out of sync TODO fi sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=${DROPGZ} INSTALL_CNS=true INSTALL_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} retryCountOnTaskFailure: 3 diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 239b853e78..373bbd0248 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -1,7 +1,6 @@ parameters: name: "" displayName: "" - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: "" clusterType: "" clusterName: "" @@ -11,16 +10,27 @@ parameters: dependsOn: "" stages: - - template: ../../templates/create-cluster.yaml - parameters: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }} - vmSize: ${{ parameters.vmSize }} - k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} - dependsOn: ${{ parameters.dependsOn }} + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_AKS_CLUSTER_TEST) - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} @@ -31,6 +41,7 @@ stages: variables: TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] condition: and(succeeded(), eq(variables.TAG, variables.CURRENT_VERSION)) jobs: - job: ${{ parameters.name }} @@ -50,4 +61,4 @@ stages: parameters: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 6719ebe25e..4af3a591da 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -29,7 +29,8 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} ls -lah pwd echo "installing kubectl" diff --git a/.pipelines/submodules-pipeline.yaml b/.pipelines/submodules-pipeline.yaml index 72a26856cd..aae1822d6b 100644 --- a/.pipelines/submodules-pipeline.yaml +++ b/.pipelines/submodules-pipeline.yaml @@ -42,6 +42,7 @@ stages: - script: | BUILD_NUMBER=$(Build.BuildNumber) echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" + echo "##vso[task.setvariable variable=commitID;isOutput=true]$(make revision)" echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)" echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" @@ -248,7 +249,6 @@ stages: parameters: name: "cilium_e2e" displayName: Cilium - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true clusterType: cilium-podsubnet-up clusterName: "submodciliume2e" @@ -261,7 +261,6 @@ stages: parameters: name: "cilium_overlay_e2e" displayName: Cilium on AKS Overlay - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true clusterType: cilium-overlay-up clusterName: "submodoverlaye2e" @@ -274,11 +273,9 @@ stages: parameters: name: "aks_swift_e2e" displayName: AKS Swift - pipelineBuildImage: "$(BUILD_IMAGE)" testDropgz: true clusterType: swift-byocni-up clusterName: "submodswift" - osSku: "Ubuntu" vmSize: Standard_B2ms k8sVersion: "" windowsOsSku: "" @@ -288,9 +285,12 @@ stages: displayName: Delete Clusters condition: always() dependsOn: + - setup - "aks_swift_e2e" - "cilium_e2e" - "cilium_overlay_cilium_e2e" + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: - job: delete displayName: Delete Cluster @@ -301,23 +301,21 @@ stages: cilium_e2e: name: cilium_e2e clusterName: 'ciliume2e' + region: $(REGION_AKS_CLUSTER_TEST) cilium_overlay_cilium_e2e: name: cilium_overlay_cilium_e2e clusterName: 'overlaye2e' + region: $(REGION_OVERLAY_CLUSTER_TEST) aks_swift_e2e: name: aks_swift_e2e clusterName: 'swifte2e' - aks_ubuntu_22_linux_e2e: - name: aks_ubuntu_22_linux_e2e - clusterName: 'ubuntu22e2e' - aks_windows_22_e2e: - name: aks_windows_22_e2e - clusterName: 'win22e2e' + region: $(REGION_OVERLAY_CLUSTER_TEST) steps: - template: templates/delete-cluster.yaml parameters: name: $(name) - clusterName: $(clusterName) + clusterName: $(clusterName)-$(commitID) + region: $(region) - stage: cleanup displayName: Cleanup diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index 2c665b9a37..1ac4b9ae01 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -2,32 +2,27 @@ parameters: name: "" displayName: "" clusterType: "" - clusterName: "" + clusterName: "" # Recommended to pass in unique identifier vmSize: "" k8sVersion: "" windowsOsSku: "" dependsOn: "" + region: "" -stages: - - stage: ${{ parameters.clusterName }} - displayName: Create Cluster - ${{ parameters.displayName }} - dependsOn: ${{ parameters.dependsOn }} - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - jobs: - - job: ${{ parameters.name }} - displayName: Cluster - ${{ parameters.name }} - steps: - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - mkdir -p ~/.kube/ - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} - echo "Cluster successfully created" - displayName: Cluster - ${{ parameters.clusterType }} +jobs: + - job: ${{ parameters.name }} + displayName: Cluster - ${{ parameters.name }} + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + mkdir -p ~/.kube/ + make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} + make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }} K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" + displayName: Cluster - ${{ parameters.clusterType }} diff --git a/.pipelines/templates/delete-cluster.yaml b/.pipelines/templates/delete-cluster.yaml index a68c5d7ea4..a95eef6a10 100644 --- a/.pipelines/templates/delete-cluster.yaml +++ b/.pipelines/templates/delete-cluster.yaml @@ -1,6 +1,7 @@ parameters: name: "" clusterName: "" + region: "" steps: - task: AzureCLI@1 @@ -10,11 +11,10 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - set -e echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks down AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }} echo "Cluster and resources down" name: delete displayName: Delete - ${{ parameters.name }} From 8aefa96d4970734d4a0b3d573c6e8c4b416cabcb Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 2 Aug 2023 07:40:29 -0700 Subject: [PATCH 12/13] Addressing Comments --- .pipelines/pipeline.yaml | 9 ++------- .../azure-cni-overlay-e2e-job-template.yaml | 2 +- .../cilium-overlay-e2e-job-template.yaml | 2 +- .pipelines/submodules-pipeline.yaml | 18 ++++++++---------- hack/aks/Makefile | 2 +- hack/aks/README.md | 7 ++++++- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 88db771282..a03e6e4ef7 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -32,6 +32,7 @@ stages: name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - script: | + # To use the variables below, you must make the respective stage's dependsOn have - setup or it will not retain context of this stage BUILD_NUMBER=$(Build.BuildNumber) echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" echo "##vso[task.setvariable variable=commitID;isOutput=true]$(make revision)" @@ -417,33 +418,27 @@ stages: cilium_e2e: name: cilium_e2e clusterName: 'ciliume2e' - region: $(REGION_AKS_CLUSTER_TEST) cilium_overlay_cilium_e2e: name: cilium_overlay_cilium_e2e clusterName: 'overlaye2e' - region: $(REGION_OVERLAY_CLUSTER_TEST) azure_cni_overlay_e2e: name: azure_cni_overlay_e2e clusterName: 'azurecnioverlaye2e' - region: $(REGION_OVERLAY_CLUSTER_TEST) aks_swift_e2e: name: aks_swift_e2e clusterName: 'swifte2e' - region: $(REGION_AKS_CLUSTER_TEST) aks_ubuntu_22_linux_e2e: name: aks_ubuntu_22_linux_e2e clusterName: 'ubuntu22e2e' - region: $(REGION_AKS_CLUSTER_TEST) aks_windows_22_e2e: name: aks_windows_22_e2e clusterName: 'win22e2e' - region: $(REGION_AKS_CLUSTER_TEST) steps: - template: templates/delete-cluster.yaml parameters: name: $(name) clusterName: $(clusterName)-$(commitID) - region: $(region) + region: $(REGION_AKS_CLUSTER_TEST) - stage: validate2 diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml index fe38a1f2ae..0cdf14fd09 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -30,7 +30,7 @@ stages: k8sVersion: ${{ parameters.k8sVersion }} windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} - region: $(REGION_OVERLAY_CLUSTER_TEST) + region: $(REGION_AKS_CLUSTER_TEST) - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index 5f007f8961..3007375f57 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -30,7 +30,7 @@ stages: k8sVersion: ${{ parameters.k8sVersion }} windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} - region: $(REGION_OVERLAY_CLUSTER_TEST) + region: $(REGION_AKS_CLUSTER_TEST) - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} diff --git a/.pipelines/submodules-pipeline.yaml b/.pipelines/submodules-pipeline.yaml index aae1822d6b..348a910dd9 100644 --- a/.pipelines/submodules-pipeline.yaml +++ b/.pipelines/submodules-pipeline.yaml @@ -40,6 +40,7 @@ stages: name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - script: | + # To use the variables below, you must make the respective stage's dependsOn have - setup or it will not retain context of this stage BUILD_NUMBER=$(Build.BuildNumber) echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" echo "##vso[task.setvariable variable=commitID;isOutput=true]$(make revision)" @@ -288,7 +289,7 @@ stages: - setup - "aks_swift_e2e" - "cilium_e2e" - - "cilium_overlay_cilium_e2e" + - "cilium_overlay_e2e" variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: @@ -300,22 +301,19 @@ stages: matrix: cilium_e2e: name: cilium_e2e - clusterName: 'ciliume2e' - region: $(REGION_AKS_CLUSTER_TEST) - cilium_overlay_cilium_e2e: - name: cilium_overlay_cilium_e2e - clusterName: 'overlaye2e' - region: $(REGION_OVERLAY_CLUSTER_TEST) + clusterName: 'submodciliume2e' + cilium_overlay_e2e: + name: cilium_overlay_e2e + clusterName: 'submodoverlaye2e' aks_swift_e2e: name: aks_swift_e2e - clusterName: 'swifte2e' - region: $(REGION_OVERLAY_CLUSTER_TEST) + clusterName: 'submodswift' steps: - template: templates/delete-cluster.yaml parameters: name: $(name) clusterName: $(clusterName)-$(commitID) - region: $(region) + region: $(REGION_AKS_CLUSTER_TEST) - stage: cleanup displayName: Cleanup diff --git a/hack/aks/Makefile b/hack/aks/Makefile index a5fbd05959..4c55e23d7e 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -99,7 +99,7 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --yes @$(MAKE) set-kubeconf -overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster +overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ diff --git a/hack/aks/README.md b/hack/aks/README.md index 1396672174..b28b83c815 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -24,11 +24,16 @@ AKS Clusters byocni-up Alias to swift-byocni-up cilium-up Alias to swift-cilium-up up Alias to swift-up - overlay-up Bring up an Overlay AzCNI cluster + overlay-byocni-up Bring up a Overlay BYO CNI cluster + overlay-cilium-up Bring up a Overlay Cilium cluster + overlay-up Bring up a Overlay AzCNI cluster swift-byocni-up Bring up a SWIFT BYO CNI cluster swift-cilium-up Bring up a SWIFT Cilium cluster swift-up Bring up a SWIFT AzCNI cluster + cilium-overlay-up Bring up a Overlay BYO CNI cluster without kube-proxy for Cilium + cilium-podsubnet-up Bring up a SWIFT BYO CNI cluster without kube-proxy for Cilium windows-cniv1-up Bring up a Windows AzCNIv1 cluster + linux-cniv1-up Bring up a Linux AzCNIv1 cluster dualstack-overlay-byocni-up Bring up an dualstack overlay cluster without CNS and CNI installed down Delete the cluster vmss-restart Restart the nodes of the cluster From e2396eb581c181b42f6aa15e04d292e31e8f1bc7 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Thu, 10 Aug 2023 11:13:04 -0700 Subject: [PATCH 13/13] Refactor dualstack from #2098 --- .../cni/k8s-e2e/k8s-e2e-job-template.yaml | 2 +- .pipelines/pipeline.yaml | 16 ++--- .../aks-swift/e2e-job-template.yaml | 4 -- .../singletenancy/aks/e2e-job-template.yaml | 7 -- .../singletenancy/aks/e2e-step-template.yaml | 8 +-- .../azure-cni-overlay-e2e-job-template.yaml | 2 - .../cilium-overlay-e2e-job-template.yaml | 2 - .../cilium-overlay-e2e-step-template.yaml | 2 +- .../cilium/cilium-e2e-job-template.yaml | 2 - .../dualstackoverlay-e2e-job-template.yaml | 70 ++++++++++--------- .../dualstackoverlay-e2e-step-template.yaml | 36 ++++------ .pipelines/templates/create-cluster.yaml | 10 ++- 12 files changed, 69 insertions(+), 92 deletions(-) diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml index 9698dd4da6..12ba7dd6e4 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml @@ -24,7 +24,7 @@ jobs: # sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx" # pulling k8s version from AKS. - eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }}-$(make revision) -n ${{ parameters.clusterName }}-$(make revision) --query "currentKubernetesVersion") + eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }} -n ${{ parameters.clusterName }} --query "currentKubernetesVersion") echo $k8sVersion curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index a03e6e4ef7..ecded34db8 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -320,7 +320,6 @@ stages: clusterName: "ciliume2e" vmSize: Standard_B2ms k8sVersion: "" - windowsOsSku: "" dependsOn: 'containerize' - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -332,7 +331,6 @@ stages: clusterName: "overlaye2e" vmSize: Standard_B2ms k8sVersion: "" - windowsOsSku: "" dependsOn: 'containerize' - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -344,7 +342,6 @@ stages: clusterName: "azurecnioverlaye2e" vmSize: Standard_B2ms k8sVersion: "" - windowsOsSku: "" dependsOn: 'containerize' - template: singletenancy/aks-swift/e2e-job-template.yaml @@ -356,7 +353,6 @@ stages: clusterName: "swifte2e" vmSize: Standard_B2s k8sVersion: "" - windowsOsSku: "" dependsOn: 'containerize' - template: singletenancy/aks/e2e-job-template.yaml @@ -381,7 +377,6 @@ stages: clusterType: windows-cniv1-up clusterName: 'win22e2e' vmSize: Standard_B2ms - windowsOsSku: 'Windows2022' os_version: 'ltsc2022' scaleup: 50 dependsOn: 'containerize' @@ -391,9 +386,11 @@ stages: name: "dualstackoverlay_e2e" displayName: AKS DualStack Overlay os: linux - testDropgz: true + clusterType: dualstack-overlay-byocni-up clusterName: "dualstackoverlaye2e" - + vmSize: Standard_B2s + dependsOn: 'containerize' + testDropgz: true - stage: delete displayName: Delete Clusters @@ -406,6 +403,7 @@ stages: - "cilium_overlay_cilium_e2e" - "aks_ubuntu_22_linux_e2e" - "aks_windows_22_e2e" + - "dualstackoverlay_e2e" variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: @@ -433,6 +431,9 @@ stages: aks_windows_22_e2e: name: aks_windows_22_e2e clusterName: 'win22e2e' + dualstackoverlay_e2e: + name: dualstackoverlay_e2e + clusterName: 'dualstackoverlaye2e' steps: - template: templates/delete-cluster.yaml parameters: @@ -440,7 +441,6 @@ stages: clusterName: $(clusterName)-$(commitID) region: $(REGION_AKS_CLUSTER_TEST) - - stage: validate2 displayName: Validate Tags dependsOn: diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index adab194e2e..220c69cddf 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -6,11 +6,8 @@ parameters: clusterName: "" vmSize: "" k8sVersion: "" - windowsOsSku: "" dependsOn: "" - - stages: - stage: ${{ parameters.clusterName }} displayName: Create Cluster - ${{ parameters.displayName }} @@ -30,7 +27,6 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index b2e624d406..bdbba2587d 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -7,12 +7,10 @@ parameters: clusterName: "" vmSize: "" k8sVersion: "" - windowsOsSku: "" os_version: "" scaleup: "" dependsOn: "" - stages: - stage: ${{ parameters.clusterName }} displayName: Create Cluster - ${{ parameters.displayName }} @@ -32,7 +30,6 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) @@ -61,13 +58,9 @@ stages: - template: e2e-step-template.yaml parameters: name: ${{ parameters.name }} - clusterType: ${{ parameters.clusterType }} clusterName: ${{ parameters.clusterName }}-$(commitID) - vmSize: ${{ parameters.vmSize }} arch: ${{ parameters.arch }} os: ${{ parameters.os }} - k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} os_version: ${{ parameters.os_version }} version: $(dropgzVersion) cniVersion: $(cniVersion) diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index f9f43069d8..04b13a4ed6 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -1,14 +1,8 @@ parameters: name: "" - clusterType: "" clusterName: "" - nodeCount: "" - vmSize: "" - k8sVersion: "" - version: "" + arch: "" os: "" - windowsOsSku: "" - cniVersion: "" os_version: "" scaleup: "" diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml index 0cdf14fd09..389a21f617 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -6,7 +6,6 @@ parameters: clusterName: "" vmSize: "" k8sVersion: "" - windowsOsSku: "" dependsOn: "" stages: @@ -28,7 +27,6 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index 3007375f57..66562ac2a5 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -6,7 +6,6 @@ parameters: clusterName: "" vmSize: "" k8sVersion: "" - windowsOsSku: "" dependsOn: "" stages: @@ -28,7 +27,6 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 4e5caa1da9..e2a89174bc 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -94,7 +94,7 @@ steps: then CNS=$(CNS_VERSION) DROPGZ=$(DROP_GZ_VERSION) && echo "Running nightly" else - CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version) # These could become out of sync TODO + CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version) fi sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=${DROPGZ} INSTALL_CNS=true INSTALL_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} retryCountOnTaskFailure: 3 diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 373bbd0248..357b31caa2 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -6,7 +6,6 @@ parameters: clusterName: "" vmSize: "" k8sVersion: "" - windowsOsSku: "" dependsOn: "" stages: @@ -28,7 +27,6 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} - windowsOsSku: ${{ parameters.windowsOsSku }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml index 56e07d99d0..16c72062d9 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml @@ -2,36 +2,64 @@ parameters: name: "" displayName: "" testDropgz: "" + clusterType: "" clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement + - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} - dependsOn: + dependsOn: - setup - publish + - ${{ parameters.clusterName }} + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: - job: ${{ parameters.name }} displayName: DualStack Overlay Test Suite - (${{ parameters.name }}) pool: name: $(BUILD_POOL_NAME_DEFAULT) - demands: + demands: + - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path - GOBIN: "$(GOPATH)/bin" # Go binaries path - modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" steps: - template: dualstackoverlay-e2e-step-template.yaml parameters: name: ${{ parameters.name }} testDropgz: ${{ parameters.testDropgz }} - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - clusterName: ${{ parameters.clusterName }} + clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} dependsOn: ${{ parameters.name }} dualstack: true @@ -39,29 +67,3 @@ stages: portforward: true hostport: true service: true - - - job: cleanup - displayName: "Cleanup" - dependsOn: - - ${{ parameters.name }} - - "cni_k8se2e" - - "dualstackoverlay_e2e" - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - condition: always() - steps: - - task: AzureCLI@1 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - make -C ./hack/aks down AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - name: "CleanupDualStackOverlaycluster" - displayName: "Cleanup DualStack Overlay Cluster" \ No newline at end of file diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index e988065ca1..ca9fec523e 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -2,6 +2,7 @@ parameters: name: "" testDropgz: "" clusterName: "" + cni: "dualstack" steps: - bash: | @@ -14,6 +15,7 @@ steps: echo '##vso[task.prependpath]$(GOROOT)/bin' name: "GoEnv" displayName: "Set up the Go environment" + - task: AzureCLI@2 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) @@ -21,27 +23,15 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - echo "Check az version" - az version - echo "Install az cli extension preview" - az extension add --name aks-preview - az extension update --name aks-preview - mkdir -p ~/.kube/ - echo "Create AKS DualStack Overlay cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) - make -C ./hack/aks dualstack-overlay-byocni-up AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Dualstack Overlay Cluster is successfully created" - displayName: Create DualStackOverlay cluster - condition: succeeded() - - - script: | - ls -lah - pwd - echo "installing kubectl" - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - kubectl cluster-info - kubectl get po -owide -A + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get po -owide -A retryCountOnTaskFailure: 3 name: "installKubectl" displayName: "Install Kubectl" @@ -68,11 +58,11 @@ steps: echo "DualStack Overlay Linux control plane Load test" sudo go test -timeout 30m -tags load -run ^TestLoad$ -tags=load echo "DualStack Overlay Linux control plane CNS validation test" - sudo go test -timeout 30m -tags load -cni dualstack -run ^TestValidateState$ -tags=load + make test-validate-state OS=${{ parameters.os }} CNI_TYPE=${{ parameters.cni }} cd ../datapath echo "Dualstack Overlay Linux datapath IPv6 test" sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true echo "Dualstack Overlay Linux datapath IPv4 test" sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration name: "DualStack_Overlay_Linux_Tests" - displayName: "DualStack Overlay Linux Tests" \ No newline at end of file + displayName: "DualStack Overlay Linux Tests" diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index 1ac4b9ae01..0ba0f03623 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -5,7 +5,7 @@ parameters: clusterName: "" # Recommended to pass in unique identifier vmSize: "" k8sVersion: "" - windowsOsSku: "" + windowsOsSku: "Windows2022" # Currently we only support Windows2022 dependsOn: "" region: "" @@ -21,6 +21,14 @@ jobs: addSpnToEnvironment: true inlineScript: | set -e + echo "Check az version" + az version + if ${{ lower(contains(parameters.clusterType, 'dualstack')) }} + then + echo "Install az cli extension preview" + az extension add --name aks-preview + az extension update --name aks-preview + fi mkdir -p ~/.kube/ make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }} K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD}