functional_test_v2.yaml

name: functional-tests

on:
  pull_request:
  push:
    branches: [ main ]
  workflow_dispatch:
    inputs:
      UPLOAD_UPDATED_EXPECTED_RESULTS:
        description: 'Set this to true to upload updated golden file expected results and upload these results as a Github workflow artifact.'
        required: false
        default: false
      UPLOAD_KUBERNETES_DEBUG_INFO:
        description: 'Set this to true to collect the debug info of the k8s cluster and upload this info as a Github workflow artifact.'
        required: false
        default: false

env:
  # Make sure to exit early if cache segment download times out after 2 minutes.
  # We limit cache download as a whole to 5 minutes.
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
jobs:
  kubernetes-test:
    env:
      KUBECONFIG: /tmp/kube-config-splunk-otel-collector-chart-functional-testing
      KUBE_TEST_ENV: kind
      UPLOAD_UPDATED_EXPECTED_RESULTS: ${{ github.event.inputs.UPLOAD_UPDATED_EXPECTED_RESULTS || 'false' }}
      UPLOAD_KUBERNETES_DEBUG_INFO: ${{ github.event.inputs.UPLOAD_KUBERNETES_DEBUG_INFO || 'false' }}
    strategy:
      fail-fast: false
      matrix:
        # The kubernetes_version matrix entries are currently following native Kubernetes version support and +1, see: https://endoflife.date/kubernetes
        # To check latest versions, see: https://hub.docker.com/r/kindest/node/tags
        k8s-version:
          - v1.30.0 # Support: 28 Apr 2025 - 28 Jun 2025
          - v1.29.0 # Support: 28 Dec 2024 - 28 Feb 2025
          - v1.28.0 # Support: 28 Aug 2024 - 28 Oct 2024
          - v1.27.3  # Support: 28 Apr 2024 - 28 Jun 2024
          - v1.26.6 # Support: 28 Dec 2023 - 28 Feb 2024
          # Test current +1 out-of-date Kubernetes version to cover EKS's extended support version matrix
          - v1.25.11 # Support: 27 Aug 2023 - 27 Oct 2023
        test-job:
          - functional
          - histogram
          - configuration_switching
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.22.5
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Create kind cluster
        uses: helm/kind-action@v1.12.0
        with:
          node_image: kindest/node:${{ matrix.k8s-version }}
          kubectl_version: ${{ matrix.k8s-version }}
          cluster_name: kind
          config: ./.github/workflows/configs/kind-config.yaml
      - name: Fix kubelet TLS server certificates
        run: |
          kubectl get csr -o=jsonpath='{range.items[?(@.spec.signerName=="kubernetes.io/kubelet-serving")]}{.metadata.name}{" "}{end}' | xargs kubectl certificate approve
      - name: Update dependencies
        run: |
          make dep-update
      - name: Deploy cert-manager
        run: |
          make cert-manager
      - name: run functional tests
        id: run-functional-tests
        env:
          K8S_VERSION: ${{ matrix.k8s-version }}
        run: |
          cd functional_tests
          TEARDOWN_BEFORE_SETUP=true UPDATE_EXPECTED_RESULTS=${{ env.UPLOAD_UPDATED_EXPECTED_RESULTS }} go test -v -tags ${{ matrix.test-job }}
      - name: Collect Kubernetes Cluster debug info on failure
        if: always() && (steps.run-functional-tests.outcome == 'failure' || env.UPLOAD_KUBERNETES_DEBUG_INFO == 'true')
        id: collect-debug-info
        run: |
          echo "Functional tests failed. Collecting debug info for current state of the Kubernetes cluster..."
          cd tools
          ./splunk_kubernetes_debug_info.sh
      - name: Upload Kubernetes Cluster debug info
        if: always() && (steps.run-functional-tests.outcome == 'failure' || env.UPLOAD_KUBERNETES_DEBUG_INFO == 'true')
        uses: actions/upload-artifact@v4
        with:
          name: k8s-debug-info-${{ matrix.test-job }}-${{ matrix.k8s-version }}
          path: tools/splunk_kubernetes_debug_info_*
          retention-days: 5
      - name: Upload test results
        if: always() && env.UPLOAD_UPDATED_EXPECTED_RESULTS == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: functional_tests-${{ matrix.test-job }}-${{ matrix.k8s-version }}
          path: functional_tests/results
          retention-days: 5

  eks-test:
    name: Test helm install in EKS - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: eks-access
    env:
      KUBECONFIG: /tmp/kube-config-splunk-otel-collector-chart-functional-testing-eks
      KUBE_TEST_ENV: eks
      SKIP_TESTS: "true" # we need to skip functional tests as we have not set probes to listen to the traffic.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_KEY }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET }}
          aws-region: us-west-1
      - name: Install kubeconfig
        run: |
          aws eks update-kubeconfig --name rotel-eks --region us-west-1
      - name: Update dependencies
        run: |
          make dep-update
      - name: Deploy cert-manager
        run: |
          make cert-manager
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
        run: |
          cd functional_tests
          TEARDOWN_BEFORE_SETUP=true go test -v -tags functional
  eks-upgrade-test:
    name: Test helm upgrade in EKS - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: eks-access
    env:
      KUBECONFIG: /tmp/kube-config-splunk-otel-collector-chart-functional-testing-eks
      KUBE_TEST_ENV: eks
      SKIP_TESTS: "true" # we need to skip functional tests as we have not set probes to listen to the traffic.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Checkout main if in a PR
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v4
        with:
          path: base
          ref: main
      - name: Checkout previous main commit if in main
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v4
        with:
          path: base
          ref: HEAD
          fetch-depth: 2
      - name: Checkout previous main commit if in main
        if: github.event_name != 'pull_request'
        run: cd base && git checkout HEAD^
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_KEY }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET }}
          aws-region: us-west-1
      - name: Install kubeconfig
        run: |
          aws eks update-kubeconfig --name rotel-eks --region us-west-1
      - name: Update dependencies
        run: |
          cd base && make dep-update
      - name: Deploy cert-manager
        run: |
          cd base && make cert-manager
      - name: Deploy previous version of the chart
        run: |
          cd base && helm install sock helm-charts/splunk-otel-collector --set cloudProvider=aws --set distribution=eks --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx
      - name: Update dependencies
        run: |
          make dep-update
      - name: Deploy cert-manager
        run: |
          make cert-manager
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
        run: |
          cd functional_tests
          go test -v -tags functional

  eks-upgrade-from-release-test:
    name: Test helm upgrade from release in EKS - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: eks-access
    env:
      KUBECONFIG: /tmp/kube-config-splunk-otel-collector-chart-functional-testing-eks
      KUBE_TEST_ENV: eks
      SKIP_TESTS: "true" # we need to skip functional tests as we have not set probes to listen to the traffic.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Checkout old release
        uses: actions/checkout@v4
        with:
          path: base
          ref: splunk-otel-collector-0.109.0
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_KEY }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET }}
          aws-region: us-west-1
      - name: Install kubeconfig
        run: |
          aws eks update-kubeconfig --name rotel-eks --region us-west-1
      - name: Update dependencies
        run: |
          cd base && make dep-update
      - name: Deploy cert-manager
        run: |
          cd base && make cert-manager
      - name: Deploy previous version of the chart
        run: |
          cd base && helm install sock helm-charts/splunk-otel-collector --set cloudProvider=aws --set distribution=eks --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx --set operator.enabled=true --set environment=dev
      - name: Update dependencies
        run: |
          make dep-update
      - name: Deploy cert-manager
        run: |
          make cert-manager
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
        run: |
          cd functional_tests
          go test -v -tags functional
  gke-autopilot-test:
    name: Test helm install in GKE/Autopilot - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: gke-autopilot-access
    env:
      KUBE_TEST_ENV: gke/autopilot
      SKIP_TESTS: "true"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - uses: 'google-github-actions/auth@v2.1.7'
        with:
          project_id: ${{ secrets.GKE_PROJECT }}
          credentials_json: ${{ secrets.GKE_SA_KEY }}
      - uses: google-github-actions/setup-gcloud@v2.1.2
        with:
          project_id: ${{ secrets.GKE_PROJECT }}
      - uses: google-github-actions/get-gke-credentials@v2.3.0
        with:
          cluster_name: ${{ secrets.GKE_AUTOPILOT_CLUSTER }}
          location: ${{ secrets.GKE_REGION }}
          project_id: ${{ secrets.GKE_PROJECT }}
      - name: Update dependencies
        run: |
          make dep-update
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
        run: |
          cd functional_tests
          TEARDOWN_BEFORE_SETUP=true go test -v -tags functional

  gke-autopilot-upgrade-test:
    name: Test helm upgrade in GKE/Autopilot - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: gke-autopilot-access
    env:
      KUBE_TEST_ENV: gke/autopilot
      SKIP_TESTS: "true"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Checkout main if in a PR
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v4
        with:
          path: base
          ref: main
      - name: Checkout previous main commit if in main
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v4
        with:
          path: base
          ref: HEAD
          fetch-depth: 2
      - name: Checkout previous main commit if in main
        if: github.event_name != 'pull_request'
        run: cd base && git checkout HEAD^
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - uses: 'google-github-actions/auth@v2.1.7'
        with:
          project_id: ${{ secrets.GKE_PROJECT }}
          credentials_json: ${{ secrets.GKE_SA_KEY }}
      - uses: google-github-actions/setup-gcloud@v2.1.2
        with:
          project_id: ${{ secrets.GKE_PROJECT }}
      - uses: google-github-actions/get-gke-credentials@v2.3.0
        with:
          cluster_name: ${{ secrets.GKE_AUTOPILOT_CLUSTER }}
          location: ${{ secrets.GKE_REGION }}
          project_id: ${{ secrets.GKE_PROJECT }}
      - name: Update dependencies
        run: |
          make dep-update
      - name: Deploy previous version of the chart
        run: |
          cd base
          helm dependency update helm-charts/splunk-otel-collector
          helm install sock helm-charts/splunk-otel-collector --set cloudProvider=gcp --set distribution=gke/autopilot --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
        run: |
          cd functional_tests
          go test -v -tags functional

  aks-windows-test:
    name: Test helm install in AKS - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: aks-windows-group
    env:
      KUBE_TEST_ENV: aks
      SKIP_TESTS: "true"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Set up kubelogin for non-interactive login
        uses: azure/use-kubelogin@v1.2
        with:
          kubelogin-version: "v0.0.24"
      - uses: azure/login@v2.2.0
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - uses: azure/aks-set-context@v4
        with:
          resource-group: ${{ secrets.AKS_RESOURCE_GROUP }}
          cluster-name: ${{ secrets.AKS_CLUSTER_NAME }}
          admin: false
          use-kubelogin: true
      - name: Update dependencies
        run: |
          make dep-update
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
        run: |
          cd functional_tests
          TEARDOWN_BEFORE_SETUP=true go test -v -tags functional

  gce-autopilot-test:
    name: Test helm install in GCE (kops) - credentials needed
    needs: kubernetes-test
    if: github.event.pull_request.head.repo.full_name == github.repository
    concurrency:
      group: gce-access
    env:
      KUBE_TEST_ENV: gce
      SKIP_TESTS: "true"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.21.8
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Update dependencies
        run: |
          make dep-update
      - name: Set kubeconfig
        run: echo "$GCE_KUBECONFIG" > /tmp/kubeconfig
        env:
          GCE_KUBECONFIG: ${{ secrets.GCE_KUBECONFIG }}
      - name: run functional tests
        env:
          HOST_ENDPOINT: 0.0.0.0
          KUBECONFIG: /tmp/kubeconfig
        run: |
          cd functional_tests
          TEARDOWN_BEFORE_SETUP=true go test -v -tags functional

  kubernetes-test-upgrade-with-no-values-change:
    name: Test helm upgrade in kind without any values change by a user
    if: github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Checkout old release
        uses: actions/checkout@v4
        with:
          path: base
          ref: splunk-otel-collector-0.113.0
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: ~1.22.5
          cache: false
      - name: Cache Go
        id: go-cache
        timeout-minutes: 5
        uses: actions/cache@v4
        with:
          path: |
            ~/go/bin
            ~/go/pkg/mod
          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
      - name: Create kind cluster
        uses: helm/kind-action@v1.12.0
        with:
          node_image: kindest/node:v1.30.0
          kubectl_version: v1.30.0
          cluster_name: kind
          config: ./.github/workflows/configs/kind-config.yaml
      - name: Fix kubelet TLS server certificates
        run: |
          kubectl get csr -o=jsonpath='{range.items[?(@.spec.signerName=="kubernetes.io/kubelet-serving")]}{.metadata.name}{" "}{end}' | xargs kubectl certificate approve
      - name: Update dependencies
        run: |
          cd base && make dep-update
      - name: Deploy cert-manager
        run: |
          cd base && make cert-manager
      - name: install helm chart with operator enabled
        run: |
          cd base && helm install sock helm-charts/splunk-otel-collector --debug \
          --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx \
          --set operator.enabled=true --set environment=dev --set clusterName=dev
        continue-on-error: true
      - name: check crds are installed
        run: |
          kubectl get crds | grep 'opentelemetry.io'

          echo "get instrumentations - none if install failed"
          kubectl get instrumentations.opentelemetry.io -A
      - name: check operator
        run: |
          kubectl describe po -lapp.kubernetes.io/name=operator
          kubectl get po -lapp.kubernetes.io/name=operator -o=name | xargs -n 1 kubectl wait --for=condition=Ready --timeout=2m
          kubectl describe po -lapp.kubernetes.io/name=operator
        continue-on-error: true
      - name: upgrade old again
        run: |
          cd base && helm upgrade sock helm-charts/splunk-otel-collector --debug \
          --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx \
          --set operator.enabled=true --set environment=dev --set clusterName=dev
        continue-on-error: true
      - name: check helm release status
        run: |
          helm status sock
      - name: check instrumentations installed
        run: |
          kubectl describe instrumentations.opentelemetry.io -A
      - name: Update dependencies
        run: |
          make dep-update
      - name: try to import instrumentation in helm
        run: |
          NAMESPACE=default
          RELEASE_NAME=sock
          kubectl get -n $NAMESPACE instrumentations.opentelemetry.io -o name | xargs -I % kubectl annotate -n $NAMESPACE % meta.helm.sh/release-name=$RELEASE_NAME
          kubectl get -n $NAMESPACE instrumentations.opentelemetry.io -o name | xargs -I % kubectl annotate -n $NAMESPACE % meta.helm.sh/release-namespace=$NAMESPACE
      - name: upgrade helm chart
        run: |
          helm upgrade sock helm-charts/splunk-otel-collector --debug \
          --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx \
          --set operator.enabled=true --set environment=dev --set clusterName=dev
        continue-on-error: true
      - name: check crds are installed
        run: |
          kubectl get crds | grep 'opentelemetry.io'