cilium-overlay-e2e-step-template.yaml

parameters:
  name: ""
  clusterName: ""
  testHubble: false

steps:
  - bash: |
      echo $UID
      sudo rm -rf $(System.DefaultWorkingDirectory)/*
    displayName: "Set up OS environment"

  - checkout: self

  - bash: |
      go version
      go env
      mkdir -p '$(GOBIN)'
      mkdir -p '$(GOPATH)/pkg'
      mkdir -p '$(modulePath)'
      echo '##vso[task.prependpath]$(GOBIN)'
      echo '##vso[task.prependpath]$(GOROOT)/bin'
    name: "GoEnv"
    displayName: "Set up the Go environment"

  - task: KubectlInstaller@0
    inputs:
      kubectlVersion: latest

  - task: AzureCLI@1
    inputs:
      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
      scriptLocation: "inlineScript"
      scriptType: "bash"
      addSpnToEnvironment: true
      inlineScript: |
        set -e
        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
        ls -lah
        pwd
        kubectl cluster-info
        kubectl get po -owide -A
        if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then FILE_PATH=-nightly && echo "Running nightly"; fi
        echo "deploy Cilium ConfigMap"
        kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-config.yaml
        echo "install Cilium ${CILIUM_VERSION_TAG}"
        # Passes Cilium image to daemonset and deployment
        envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
        envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
        # Use different file directories for nightly and current cilium version
        kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-agent
        kubectl apply -f test/integration/manifests/cilium/cilium${FILE_PATH}-operator
        kubectl get po -owide -A
    name: "installCilium"
    displayName: "Install Cilium on AKS Overlay"

  - script: |
      echo "install cilium CLI"
      if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then
        echo "Cilium Agent Version ${BASH_REMATCH[0]}"
        CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt)
      else
        echo "Cilium Agent Version ${CILIUM_VERSION_TAG}"
        CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
      fi
      CLI_ARCH=amd64
      if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
      curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
      sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
      sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
      rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
      cilium status
      cilium version
    name: "installCiliumCLI"
    displayName: "Install Cilium CLI"

  - script: |
      echo "install kubetest2 and gsutils"
      go get github.com/onsi/ginkgo/ginkgo
      go get github.com/onsi/gomega/...
      go install github.com/onsi/ginkgo/ginkgo@latest
      go install sigs.k8s.io/kubetest2@latest
      go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
      go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
      wget https://storage.googleapis.com/pub/gsutil.tar.gz
      tar xfz gsutil.tar.gz
      sudo mv gsutil /usr/local/bin
    name: "installKubetest"
    displayName: "Set up Conformance Tests"

  - script: |
      echo "Start Azilium E2E Tests on Overlay Cluster"
      if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]
      then
          CNS=$(CNS_VERSION) IPAM=$(AZURE_IPAM_VERSION) && echo "Running nightly"
      else
          CNS=$(make cns-version) IPAM=$(make azure-ipam-version)
      fi
      sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true
    retryCountOnTaskFailure: 3
    name: "aziliumTest"
    displayName: "Run Azilium E2E on AKS Overlay"

  - script: |
      echo "Status of the nodes and pods after the test"
      kubectl get nodes -o wide
      kubectl get pods -A -o wide
      echo "Logs will be available as a build artifact"
      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
      echo $ARTIFACT_DIR
      sudo rm -rf $ARTIFACT_DIR
      sudo mkdir $ARTIFACT_DIR
      sudo cp test/integration/logs/* $ARTIFACT_DIR
    name: "GetLogs"
    displayName: "Get logs"
    condition: always()

  - task: PublishBuildArtifacts@1
    inputs:
      artifactName: test-output
      pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
    condition: always()

  - script: |
      kubectl get pods -A
      echo "Waiting < 2 minutes for cilium to be ready"
      # Ensure Cilium is ready Xm\Xs
      cilium status --wait --wait-duration 2m
    retryCountOnTaskFailure: 3
    name: "CiliumStatus"
    displayName: "Cilium Status"

  - script: |
      echo "Run Service Conformance E2E"
      export PATH=${PATH}:/usr/local/bin/gsutil
      KUBECONFIG=~/.kube/config kubetest2 noop \
        --test ginkgo -- \
        --focus-regex "Services.*\[Conformance\].*" \
        --skip-regex "should serve endpoints on same port and different protocols" # Cilium does not support this feature. For more info on test: https://github.com/kubernetes/kubernetes/blame/e602e9e03cd744c23dde9fee09396812dd7bdd93/test/conformance/testdata/conformance.yaml#L1780-L1788
    name: "servicesConformance"
    displayName: "Run Services Conformance Tests"

  - script: |
      echo "Run Cilium Connectivity Tests"
      cilium status
      if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]
      then
        # Until the issue https://github.com/cilium/cilium/issues/29213 is resolved, skip metrics check for nightly run
        cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption,!allow-all-with-metrics-check'
      else
        cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption'
      fi
    retryCountOnTaskFailure: 3
    name: "ciliumConnectivityTests"
    displayName: "Run Cilium Connectivity Tests"

  - ${{ if eq( parameters['testHubble'], true) }}:
      - script: |
          echo "enable Hubble metrics server"
          kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml
          kubectl apply -f test/integration/manifests/cilium/cilium-config-hubble.yaml
          kubectl rollout restart ds cilium -n kube-system
          echo "wait <3 minutes for pods to be ready after restart"
          kubectl rollout status ds cilium -n kube-system --timeout=3m
          kubectl get pods -Aowide
          echo "verify Hubble metrics endpoint is usable"
          go test ./test/integration/networkobservability -count=1 -v -tags=networkobservability
        retryCountOnTaskFailure: 3
        name: "HubbleConnectivityTests"
        displayName: "Run Hubble Connectivity Tests"

  - script: |
      echo "validate pod IP assignment and check systemd-networkd restart"
      kubectl get pod -owide -A
      # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change.
      # Saves 17 minutes
      kubectl delete deploy -n cilium-test echo-external-node
      if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then
        echo "Check cilium identities in cilium-test namepsace during nightly run"
        echo "expect the identities to be deleted when the namespace is deleted"
        kubectl get ciliumidentity | grep cilium-test
      fi
      make test-validate-state
      echo "delete cilium connectivity test resources and re-validate state"
      kubectl delete ns cilium-test
      kubectl get pod -owide -A
      make test-validate-state
    name: "validatePods"
    displayName: "Validate Pods"

  - script: |
      if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then
        kubectl get pod -owide -n cilium-test
        echo "wait for pod and cilium identity deletion in cilium-test namespace"
        ns="cilium-test"
        while true; do
          pods=$(kubectl get pods -n $ns --no-headers=true 2>/dev/null)
          if [[ -z "$pods" ]]; then
            echo "No pods found"
              break
          fi
          sleep 2s
        done
        sleep 20s
        echo "Verify cilium identities are deleted from cilium-test"
        checkIdentity="$(kubectl get ciliumidentity -o json | grep cilium-test | jq -e 'length == 0')"
        if [[ -n $checkIdentity ]]; then
          echo "##[error]Cilium Identities still present in cilium-test namespace"
        else
          printf -- "Identities deleted from cilium-test namespace\n"
        fi
      else
        echo "skip cilium identities check for PR pipeline"
      fi
    name: "CiliumIdentities"
    displayName: "Verify Cilium Identities Deletion"

  - script: |
      echo "validate pod IP assignment before CNS restart"
      kubectl get pod -owide -A
      make test-validate-state
      echo "restart CNS"
      kubectl rollout restart ds azure-cns -n kube-system
      kubectl rollout status ds azure-cns -n kube-system
      kubectl get pod -owide -A
      echo "validate pods after CNS restart"
      make test-validate-state
    name: "restartCNS"
    displayName: "Restart CNS and validate pods"

  - script: |
      echo "Run wireserver and metadata connectivity Tests"
      bash test/network/wireserver_metadata_test.sh
    retryCountOnTaskFailure: 3
    name: "WireserverMetadataConnectivityTests"
    displayName: "Run Wireserver and Metadata Connectivity Tests"

  - script: |
      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
      echo $ARTIFACT_DIR
      sudo rm -rf $ARTIFACT_DIR
      sudo rm -rf test/integration/logs
    name: "Cleanupartifactdir"
    displayName: "Cleanup artifact dir"
    condition: always()