diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml deleted file mode 100644 index 8ad83b34d36d1..0000000000000 --- a/.github/workflows/serverless-benchmarks.yml +++ /dev/null @@ -1,196 +0,0 @@ -name: "Serverless Benchmarks" - -on: - pull_request: - paths: - - 'cmd/serverless/**' - - 'pkg/serverless/**' - - '.github/workflows/serverless-benchmarks.yml' - -env: - DD_API_KEY: must-be-set - -concurrency: - group: ${{ github.workflow }}/PR#${{ github.event.pull_request.number }} - cancel-in-progress: true - -permissions: {} - -jobs: - baseline: - name: Baseline - runs-on: ubuntu-latest - outputs: - sha: ${{ steps.prepare.outputs.sha }} - steps: - - name: Checkout ${{ github.base_ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - ref: ${{ github.base_ref }} - persist-credentials: false - - - name: Install Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 - with: - go-version: stable - - - name: Prepare working tree - id: prepare - run: | - echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - go get ./... - - - name: Run benchmark - env: - TEMP_RUNNER: ${{runner.temp}} - run: | - go test -tags=test -run='^$' -bench=StartEndInvocation -count=10 -benchtime=500ms -timeout=60m \ - ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log - - - name: Upload result artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: baseline.log - path: ${{runner.temp}}/benchmark.log - if-no-files-found: error - - - current: - name: Current - runs-on: ubuntu-latest - outputs: - sha: ${{ steps.prepare.outputs.sha }} - - steps: - - name: Checkout ${{ github.ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - ref: ${{ github.sha }} - persist-credentials: false - - - name: Install Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 - with: - go-version: stable - - - name: Prepare working tree - id: prepare - run: | - echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - go get ./... - - - name: Run benchmark - env: - TEMP_RUNNER: ${{runner.temp}} - run: | - go test -tags=test -run='^$' -bench=StartEndInvocation -count=10 -benchtime=500ms -timeout=60m \ - ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log - - - name: Upload result artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: current.log - path: ${{runner.temp}}/benchmark.log - if-no-files-found: error - - summary: - name: Summary - runs-on: ubuntu-latest - needs: [baseline, current] - permissions: - pull-requests: write - - steps: - - name: Install Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 - with: - go-version: stable - cache: false - - - name: Install benchstat - run: | - go install golang.org/x/perf/cmd/benchstat@latest - - - name: Download baseline artifact - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: baseline.log - path: baseline - - name: Download current artifact - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: current.log - path: current - - - name: Analyze results - id: analyze - run: | - benchstat -row /event baseline/benchmark.log current/benchmark.log | tee analyze.txt - echo "analyze<> $GITHUB_OUTPUT - cat analyze.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Post comment - uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2.9.0 - with: - header: serverless-benchmarks - recreate: true - message: | - ## Serverless Benchmark Results - - `BenchmarkStartEndInvocation` comparison between ${{ needs.baseline.outputs.sha }} and ${{ needs.current.outputs.sha }}. - -
- tl;dr - - Use these benchmarks as an insight tool during development. - - 1. Skim down the `vs base` column in each chart. If there is a `~`, then there was no statistically significant change to the benchmark. Otherwise, ensure the estimated percent change is either negative or very small. - - 2. The last row of each chart is the `geomean`. Ensure this percentage is either negative or very small. - -
- -
- What is this benchmarking? - - The [`BenchmarkStartEndInvocation`](https://github.com/DataDog/datadog-agent/blob/main/pkg/serverless/daemon/routes_test.go) compares the amount of time it takes to call the `start-invocation` and `end-invocation` endpoints. For universal instrumentation languages (Dotnet, Golang, Java, Ruby), this represents the majority of the duration overhead added by our tracing layer. - - The benchmark is run using a large variety of lambda request payloads. In the charts below, there is one row for each event payload type. - -
- -
- How do I interpret these charts? - - The charts below comes from [`benchstat`](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat). They represent the statistical change in _duration (sec/op)_, _memory overhead (B/op)_, and _allocations (allocs/op)_. - - The benchstat docs explain how to interpret these charts. - - > Before the comparison table, we see common file-level configuration. If there are benchmarks with different configuration (for example, from different packages), benchstat will print separate tables for each configuration. - > - > The table then compares the two input files for each benchmark. It shows the median and 95% confidence interval summaries for each benchmark before and after the change, and an A/B comparison under "vs base". ... The p-value measures how likely it is that any differences were due to random chance (i.e., noise). The "~" means benchstat did not detect a statistically significant difference between the two inputs. ... - > - > Note that "statistically significant" is not the same as "large": with enough low-noise data, even very small changes can be distinguished from noise and considered statistically significant. It is, of course, generally easier to distinguish large changes from noise. - > - > Finally, the last row of the table shows the geometric mean of each column, giving an overall picture of how the benchmarks changed. Proportional changes in the geomean reflect proportional changes in the benchmarks. For example, given n benchmarks, if sec/op for one of them increases by a factor of 2, then the sec/op geomean will increase by a factor of ⁿ√2. - -
- -
- I need more help - - First off, do not worry if the benchmarks are failing. They are not tests. The intention is for them to be a tool for you to use during development. - - If you would like a hand interpreting the results come chat with us in `#serverless-agent` in the internal DataDog slack or in `#serverless` in the [public DataDog slack](https://chat.datadoghq.com/). We're happy to help! - -
- -
- Benchmark stats - - ``` - ${{ steps.analyze.outputs.analyze }} - ``` - -
diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml deleted file mode 100644 index 7be692d81d51a..0000000000000 --- a/.github/workflows/serverless-binary-size.yml +++ /dev/null @@ -1,179 +0,0 @@ -name: "Serverless Binary Size" - -on: - pull_request: - paths: - - 'cmd/serverless/**' - - 'cmd/serverless-init/**' - - 'pkg/serverless/**' - -env: - SIZE_ALLOWANCE: fromJSON(1000000) # 1 MB - -permissions: {} - -jobs: - comment: - runs-on: ubuntu-latest - permissions: - pull-requests: write # Add comment to PR - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - path: go/src/github.com/DataDog/datadog-agent - persist-credentials: false - - - name: Checkout datadog-agent base branch - run: | - # on pull request, use the merge-base - # on merge queue, just use the latest main - if [ -n "$GITHUB_HEAD_REF" ]; then - cd go/src/github.com/DataDog/datadog-agent - git fetch origin $GITHUB_HEAD_REF $GITHUB_BASE_REF - TARGET=$(git merge-base origin/$GITHUB_HEAD_REF origin/$GITHUB_BASE_REF) - git checkout $TARGET - fi - - - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - repository: DataDog/datadog-lambda-extension - path: go/src/github.com/DataDog/datadog-lambda-extension - persist-credentials: false - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Previous binary size and dependencies - id: previous - run: | - cd go/src/github.com/DataDog/datadog-lambda-extension - - OUTPUT=$(./scripts/visualize_size.sh size) - echo "binary size before merging this pull request is $OUTPUT" - echo "result=$OUTPUT" >> $GITHUB_OUTPUT - - echo "deps<> $GITHUB_OUTPUT - ./scripts/visualize_size.sh list_symbols | awk '{print $2}' | head -n 100 >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Checkout datadog-agent pr branch - run: | - cd go/src/github.com/DataDog/datadog-agent - git fetch origin $GITHUB_SHA --depth 1 - git checkout $GITHUB_SHA - - - name: Current binary size and dependencies - id: current - run: | - cd go/src/github.com/DataDog/datadog-lambda-extension - - OUTPUT=$(./scripts/visualize_size.sh size) - echo "binary size after merging this pull request will be $OUTPUT" - echo "result=$OUTPUT" >> $GITHUB_OUTPUT - - echo "deps<> $GITHUB_OUTPUT - ./scripts/visualize_size.sh list_symbols | awk '{print $2}' | head -n 100 >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Compare sizes - env: - PREVIOUS_SIZE: ${{ steps.previous.outputs.result }} - CURRENT_SIZE: ${{ steps.current.outputs.result }} - id: compare - run: | - OUTPUT=$(( $CURRENT_SIZE - $PREVIOUS_SIZE )) - echo "binary size changed by $OUTPUT bytes" - echo "diff=$OUTPUT" >> $GITHUB_OUTPUT - - OUTPUT=$(( $OUTPUT / 100000 )) - echo "cold start time changed by $OUTPUT ms" - echo "coldstart=$OUTPUT" >> $GITHUB_OUTPUT - - - name: Should post comment - env: - GIT_DIFF: ${{ steps.compare.outputs.diff }} - id: should - run: | - cd go/src/github.com/DataDog/datadog-agent - git fetch origin $GITHUB_BASE_REF - git fetch origin $GITHUB_HEAD_REF - if test $( - git diff origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF --name-only | grep dependencies_linux_amd64.txt - ); then - echo "should_run=true" >> $GITHUB_OUTPUT - echo "dependencies list changed" - elif [[ $GIT_DIFF > env.SIZE_ALLOWANCE ]]; then - echo "should_run=true" >> $GITHUB_OUTPUT - echo "binary size changed" - else - echo "should_run=false" >> $GITHUB_OUTPUT - echo "nothing changed" - fi - - ### Steps below run if size diff > SIZE_ALLOWANCE or file dependencies_linux_amd64.txt changed ### - - - name: Install graphviz - uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c # v2.0.2 - if: steps.should.outputs.should_run == 'true' - - - name: Install digraph - if: steps.should.outputs.should_run == 'true' - run: | - GOPATH=$(pwd)/go go install golang.org/x/tools/cmd/digraph@latest - - - name: List new dependencies - env: - PREVIOUS_DEPS: ${{ steps.previous.outputs.deps }} - CURRENT_DEPS: ${{ steps.current.outputs.deps }} - id: deps - if: steps.should.outputs.should_run == 'true' - run: | - echo "deps<> $GITHUB_OUTPUT - for dep in $(echo "$CURRENT_DEPS"); do - if ! echo "$PREVIOUS_DEPS" | grep -w -q "$dep"; then - echo "$dep" >> $GITHUB_OUTPUT - fi - done - echo "EOF" >> $GITHUB_OUTPUT - - - name: Create dependency graphs - env: - DEPS: ${{ steps.deps.outputs.deps }} - if: steps.should.outputs.should_run == 'true' - run: | - export PATH=$(pwd)/go/bin:$PATH - cd go/src/github.com/DataDog/datadog-lambda-extension - mkdir graphs - for dep in $(echo "$DEPS"); do - PACKAGE=$dep ./scripts/visualize_size.sh graph - mv .layers/output.svg graphs/$(echo $dep | tr '/' '-').svg - done - - - name: Archive dependency graphs - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - if: steps.should.outputs.should_run == 'true' - with: - name: dependency-graphs - path: go/src/github.com/DataDog/datadog-lambda-extension/graphs - - - name: Write message - id: write - if: steps.should.outputs.should_run == 'true' - env: - VAR_COLD_START: ${{ steps.compare.outputs.coldstart }} - VAR_DIFF: ${{ steps.compare.outputs.diff }} - VAR_DEPS: ${{ steps.deps.outputs.deps }} - VAR_RUN_ID: ${{ github.run_id }} - run: | - cd go/src/github.com/DataDog/datadog-agent - ./test/integration/serverless_perf/write_message.sh - - - name: Post comment - uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2.9.0 - if: steps.should.outputs.should_run == 'true' - with: - header: serverless-binary-size - recreate: true - path: ${{ steps.write.outputs.filename }} diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml deleted file mode 100644 index ace5e88fbda98..0000000000000 --- a/.github/workflows/serverless-integration.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: "Serverless Integration Tests" - -on: - pull_request: - branches-ignore: - - 'mq-working-branch-*' - paths: - - 'cmd/serverless/**' - - 'pkg/serverless/**' - - 'test/integration/serverless/**' - - '.github/workflows/serverless-integration.yml' - - 'go.mod' - schedule: - - cron: '0 14 * * *' # cron schedule uses UTC timezone. Run tests at the beginning of the day in US-East - -permissions: {} - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - architecture: [amd64, arm64] - suite: [metric, log, trace, appsec, proxy] - name: ${{ matrix.suite }} on ${{ matrix.architecture }} - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - path: go/src/github.com/DataDog/datadog-agent - persist-credentials: false - - - name: Set up Node 20 - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 - with: - node-version: 20 - - - name: Install Serverless Framework - run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local - - - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - with: - repository: DataDog/datadog-lambda-extension - path: go/src/github.com/DataDog/datadog-lambda-extension - persist-credentials: false - - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - with: - image: tonistiigi/binfmt:latest - platforms: amd64,arm64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Create raw logs directory - id: rawlogs - run: |- - DIR=$(mktemp -d) - echo "dir=${DIR}" >> $GITHUB_OUTPUT - - - name: Run tests if AWS credentials are available - id: test - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.SERVERLESS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.SERVERLESS_AWS_SECRET_ACCESS_KEY }} - GOWORK: off - with: - timeout_minutes: 120 - max_attempts: 2 - command: | - RAWLOGS_DIR="${{ steps.rawlogs.outputs.dir }}/${{ matrix.architecture }}" - cd go/src/github.com/DataDog/datadog-agent - ARCHITECTURE=${{ matrix.architecture }} RAWLOGS_DIR=$RAWLOGS_DIR \ - ./test/integration/serverless/run.sh ${{ matrix.suite }} - - - name: Archive raw logs - if: always() - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: rawlogs-${{ matrix.suite }}-${{ matrix.architecture }} - path: ${{ steps.rawlogs.outputs.dir }} diff --git a/.github/workflows/serverless-vuln-scan.yml b/.github/workflows/serverless-vuln-scan.yml deleted file mode 100644 index b7dabe4fee30a..0000000000000 --- a/.github/workflows/serverless-vuln-scan.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: "Serverless Vulnerability Scan" - -on: - pull_request: - paths: - - 'cmd/serverless/**' - - 'cmd/serverless-init/**' - - 'pkg/serverless/**' - - '.github/workflows/serverless-vuln-scan.yml' - -env: - VERSION: 1 # env var required when building extension - -jobs: - run: - runs-on: ubuntu-latest - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@v4 - with: - path: go/src/github.com/DataDog/datadog-agent - - - name: Checkout datadog-lambda-extension repository - uses: actions/checkout@v4 - with: - repository: DataDog/datadog-lambda-extension - path: go/src/github.com/DataDog/datadog-lambda-extension - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build extension - run: | - cd go/src/github.com/DataDog/datadog-lambda-extension - ./scripts/build_binary_and_layer_dockerized.sh - - - name: Scan amd64 image with trivy - uses: aquasecurity/trivy-action@master - with: - image-ref: "datadog/build-lambda-extension-amd64:${{ env.VERSION }}" - ignore-unfixed: true - exit-code: 1 - format: table - - - name: Scan arm64 image with trivy - uses: aquasecurity/trivy-action@master - with: - image-ref: "datadog/build-lambda-extension-arm64:${{ env.VERSION }}" - ignore-unfixed: true - exit-code: 1 - format: table - - - name: Scan amd64 image with grype - uses: anchore/scan-action@v3 - with: - image: "datadog/build-lambda-extension-amd64:${{ env.VERSION }}" - only-fixed: true - fail-build: true - severity-cutoff: low - output-format: table - - - name: Scan arm64 image with grype - uses: anchore/scan-action@v3 - with: - image: "datadog/build-lambda-extension-arm64:${{ env.VERSION }}" - only-fixed: true - fail-build: true - severity-cutoff: low - output-format: table - - - name: Scan binary files with grype - uses: anchore/scan-action@v3 - with: - path: go/src/github.com/DataDog/datadog-lambda-extension/.layers - only-fixed: true - fail-build: true - severity-cutoff: low - output-format: table