signalfx · jeffreyc-splunk · May 16, 2023
@@ -136,11 +136,11 @@ updates:
     schedule:
       interval: "weekly"
   - package-ecosystem: "docker"
-    directory: "/instrumentation"
+    directory: "/instrumentation/packaging/fpm"
     schedule:
       interval: "weekly"
   - package-ecosystem: "docker"
-    directory: "/instrumentation/packaging/fpm"
+    directory: "/instrumentation/splunk-otel-auto-instrumentation"
     schedule:
       interval: "weekly"
   - package-ecosystem: "docker"

@@ -8,45 +8,54 @@ on:
     paths:
       - '.github/workflows/auto-instrumentation.yml'
       - 'instrumentation/**'
+      - 'internal/buildscripts/packaging/tests/helpers/**'
+      - 'internal/buildscripts/packaging/tests/instrumentation/**'
+      - 'internal/buildscripts/packaging/tests/requirements.txt'
+      - '!**.md'
 
 concurrency:
   group: auto-instrumentation-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  PYTHON_VERSION: '3.10'
+  PIP_VERSION: '22.0.4'
+  REQUIREMENTS_PATH: "internal/buildscripts/packaging/tests/requirements.txt"
+  GO_VERSION: 1.20.6
+
 jobs:
-  libsplunk:
+  cross-compile:
     # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        ARCH: [ "amd64", "arm64" ]
+        SYS_BINARIES: [ "binaries-linux_amd64", "binaries-linux_arm64" ]
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        if: ${{ matrix.ARCH == 'arm64' }}
-        uses: docker/setup-qemu-action@v2
+      - name: Set up Go
+        uses: actions/setup-go@v4
         with:
-          platforms: arm64
-          image: tonistiigi/binfmt:qemu-v7.0.0
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: '**/go.sum'
 
-      - name: Build libsplunk.so
-        run: make -C instrumentation dist ARCH=${{ matrix.ARCH }}
+      - name: Build Collector
+        run: |
+          make ${{ matrix.SYS_BINARIES }}
 
-      - name: Upload artifact
+      - name: Uploading binaries
         uses: actions/upload-artifact@v3
         with:
-          name: libsplunk-${{ matrix.ARCH }}
-          path: ./instrumentation/dist/libsplunk_${{ matrix.ARCH }}.so
+          name: ${{ matrix.SYS_BINARIES }}
+          path: |
+            ./bin/*
 
-  build-package:
-    name: build-package
-    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
+  instrumentation-package:
     runs-on: ubuntu-20.04
-    needs: [libsplunk]
     strategy:
       matrix:
+        PACKAGE: [ "splunk-otel-auto-instrumentation", "splunk-otel-systemd-auto-instrumentation" ]
         SYS_PACKAGE: [ "deb", "rpm" ]
         ARCH: [ "amd64", "arm64"]
     steps:
@@ -55,86 +64,111 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Downloading libsplunk-${{ matrix.ARCH }}
-        uses: actions/download-artifact@v3
+      - uses: docker/setup-qemu-action@v2
+        if: ${{ matrix.ARCH != 'amd64' }}
         with:
-          name: libsplunk-${{ matrix.ARCH }}
-          path: ./instrumentation/dist
+          platforms: ${{ matrix.ARCH }}
+          image: tonistiigi/binfmt:qemu-v7.0.0
 
-      - name: Build ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package
-        run: |
-          gitRef=(${GITHUB_REF//// })
-          if [[ "${gitRef[1]}" = "tags" ]] ;
-          then
-            echo "TAG_NAME=${gitRef[2]}" >> $GITHUB_ENV
-          else
-            echo "TAG_NAME=" >> $GITHUB_ENV
-          fi
-          make -C instrumentation ${{ matrix.SYS_PACKAGE }}-package SKIP_COMPILE=true VERSION="${TAG_NAME:-}" ARCH="${{ matrix.ARCH }}"
+      - name: Build ${{ matrix.PACKAGE }} ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package
+        run: make -C instrumentation/${{ matrix.PACKAGE }} ${{ matrix.SYS_PACKAGE }}-package ARCH="${{ matrix.ARCH }}"
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
-          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-${{ matrix.SYS_PACKAGE }}
-          path: ./instrumentation/dist/splunk-otel-auto-instrumentation*.${{ matrix.SYS_PACKAGE }}
+          name: ${{ matrix.PACKAGE }}-${{ matrix.ARCH }}-${{ matrix.SYS_PACKAGE }}
+          path: ./instrumentation/${{ matrix.PACKAGE }}/dist/${{ matrix.PACKAGE }}*.${{ matrix.SYS_PACKAGE }}
 
-  test-deb-package:
-    name: test-deb-package
-    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
+  test-instrumentation-matrix:
     runs-on: ubuntu-20.04
-    needs: [build-package]
-    strategy:
-      matrix:
-        DISTRO: [ "debian:8", "debian:9", "debian:10", "ubuntu:16.04", "ubuntu:18.04", "ubuntu:20.04" ]
-        ARCH: [ "amd64", "arm64" ]
-        exclude:
-          - DISTRO: debian:8
-            ARCH: arm64
     steps:
-      - name: Check out the codebase.
+      - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Downloading splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-deb
-        uses: actions/download-artifact@v3
-        with:
-          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-deb
-          path: ./instrumentation/dist
-
-      - name: Set up QEMU
-        if: ${{ matrix.ARCH == 'arm64' }}
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: arm64
-          image: tonistiigi/binfmt:qemu-v7.0.0
-
-      - name: Test
-        run: docker run --platform linux/${{ matrix.ARCH }} --rm -v $(pwd):/repo -w /repo ${{ matrix.DISTRO }} /repo/instrumentation/packaging/fpm/test.sh deb ${{ matrix.ARCH }}
-
-  test-rpm-package:
-    name: test-rpm-package
+      - name: Get matrix
+        id: get-matrix
+        run: |
+          # create test matrix for distro and arch
+          dockerfiles=$(find internal/buildscripts/packaging/tests/instrumentation/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u)
+          if [ -z "$dockerfiles" ]; then
+            echo "Failed to get dockerfiles from internal/buildscripts/packaging/tests/instrumentation/images!" >&2
+            exit 1
+          fi
+          distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done)
+          arch="\"amd64\", \"arm64\""
+          matrix="{\"DISTRO\": [${distro%,}], \"ARCH\": [${arch}]}"
+          echo "$matrix" | jq
+          echo "matrix=${matrix}" >> $GITHUB_OUTPUT
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+
+  test-instrumentation-package:
     # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
-    runs-on: ubuntu-20.04
-    needs: [build-package]
+    runs-on: ${{ fromJSON('["ubuntu-20.04", "ubuntu-22.04"]')[matrix.DISTRO == 'amazonlinux-2023'] }}
+    timeout-minutes: 60
+    needs: [cross-compile, instrumentation-package, test-instrumentation-matrix]
     strategy:
-      matrix:
-        DISTRO: [ "centos:7", "centos:8", "amazonlinux:2", "opensuse/leap:42", "opensuse/leap:15", "oraclelinux:7", "oraclelinux:8" ]
-        ARCH: [ "amd64", "arm64" ]
+      matrix: ${{ fromJSON(needs.test-instrumentation-matrix.outputs.matrix) }}
+      fail-fast: false
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v3
 
-      - name: Downloading splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-rpm
-        uses: actions/download-artifact@v3
+      - name: Get package type for ${{ matrix.DISTRO }}
+        run: |
+          for pkg in "deb" "rpm"; do
+            if [[ -f "internal/buildscripts/packaging/tests/instrumentation/images/${pkg}/Dockerfile.${{ matrix.DISTRO }}" ]]; then
+              echo "SYS_PACKAGE=${pkg}" >> $GITHUB_ENV
+              exit 0
+            fi
+          done
+          echo "Unknown distro '${{ matrix.DISTRO }}'!"
+          exit 1
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: binaries-linux_${{ matrix.ARCH }}
+          path: ./bin
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-${{ env.SYS_PACKAGE }}
+          path: ./instrumentation/splunk-otel-auto-instrumentation/dist
+
+      - uses: actions/download-artifact@v3
         with:
-          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-rpm
-          path: ./instrumentation/dist
+          name: splunk-otel-systemd-auto-instrumentation-${{ matrix.ARCH }}-${{ env.SYS_PACKAGE }}
+          path: ./instrumentation/splunk-otel-systemd-auto-instrumentation/dist
 
-      - name: Set up QEMU
-        if: ${{ matrix.ARCH == 'arm64'}}
-        uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v2
+        if: ${{ matrix.ARCH != 'amd64' }}
         with:
-          platforms: arm64
+          platforms: ${{ matrix.ARCH }}
           image: tonistiigi/binfmt:qemu-v7.0.0
 
-      - name: Test
-        run: docker run --platform linux/${{ matrix.ARCH }} --rm -v $(pwd):/repo -w /repo ${{ matrix.DISTRO }} /repo/instrumentation/packaging/fpm/test.sh rpm ${{ matrix.ARCH }}
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: ${{ env.REQUIREMENTS_PATH }}
+
+      - name: Install pytest
+        run: |
+          if which pip; then
+              pip install --upgrade 'pip==${{ env.PIP_VERSION }}'
+          else
+              curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+              python get-pip.py 'pip==${{ env.PIP_VERSION }}'
+          fi
+          pip install -r "${{ env.REQUIREMENTS_PATH }}"
+
+      - name: Test ${{ env.SYS_PACKAGE }} on ${{ matrix.DISTRO }} ${{ matrix.ARCH }}
+        run: |
+          distro="${{ matrix.DISTRO }}"
+          if [[ "$distro" = "amazonlinux-2" ]]; then
+            # workaround for pytest substring matching
+            distro="amazonlinux-2 and not amazonlinux-2023"
+          fi
+          python3 -u -m pytest -sx --verbose -k "$distro and ${{ matrix.ARCH }}" \
+            internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py
@@ -196,10 +196,10 @@ libsplunk:
       - ARCH: ["amd64", "arm64"]
   script:
     - *docker-reader-role
-    - make -C instrumentation dist ARCH=${ARCH} DOCKER_REPO=${DOCKER_HUB_REPO}
+    - make -C instrumentation/splunk-otel-auto-instrumentation dist ARCH=${ARCH} DOCKER_REPO=${DOCKER_HUB_REPO}
   artifacts:
     paths:
-      - instrumentation/dist/libsplunk_*.so
+      - instrumentation/splunk-otel-auto-instrumentation/dist/libsplunk_*.so
 
 agent-bundle-linux:
   rules:
@@ -255,11 +255,12 @@ agent-bundle-windows:
     - libsplunk
   parallel:
     matrix:
+      - PKG: [splunk-otel-auto-instrumentation, splunk-otel-envvar-auto-instrumentation]
       - ARCH: [amd64, arm64]
   before_script:
     - ./instrumentation/packaging/fpm/install-deps.sh
   script:
-    - ./instrumentation/packaging/fpm/${PKG_TYPE}/build.sh "${CI_COMMIT_TAG:-}" "$ARCH" "./dist"
+    - ./instrumentation/${PKG}/packaging/${PKG_TYPE}/build.sh "${CI_COMMIT_TAG:-}" "$ARCH" "./dist"
 
 instrumentation-deb:
   extends: .instrumentation-deb-rpm