Merge pull request #793 from tjungblu/nightly_bench

add nightly benchmark

.github/workflows/benchmark-pr.yaml

@@ -1,42 +1,9 @@
 ---
-name: Benchmarks on AMD64
+name: Benchmarks on PRs (AMD64)
 permissions: read-all
 on: [pull_request]
 jobs:
-  benchmark-pull-request:
-    runs-on: ubuntu-latest-8-cores
-    steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
-      with:
-        fetch-depth: 0
-    - id: goversion
-      run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
-    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-      with:
-        go-version: ${{ steps.goversion.outputs.goversion }}
-    - name: Run Benchmarks
-      run: |
-        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ github.event.pull_request.base.sha }}
-    - run: |
-        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
-        cat result.txt >> "$GITHUB_STEP_SUMMARY"
-        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
-        cat <<EOL >> "$GITHUB_STEP_SUMMARY"
-        <hr />
-        The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE of the pull request, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10).
-        The last row has the Geometric Mean (geomean) for the given rows in the table.
-        Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help.
-        EOL
-    - name: Validate results under acceptable limit
-      run: |
-        export MAX_ACCEPTABLE_DIFFERENCE=5
-        while IFS= read -r line; do
-          # Get fourth value, which is the comparison with the base.
-          value="$(echo "$line" | awk '{print $4}')"
-          if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then
-            if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then
-              echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)"
-              exit 1
-            fi
-          fi
-        done < <(grep geomean result.txt)
+  amd64:
+    uses: ./.github/workflows/benchmark-template.yaml
+    with:
+      benchGitRef: ${{ github.event.pull_request.base.sha }}

.github/workflows/benchmark-releases.yaml

@@ -0,0 +1,13 @@
+---
+name: Nightly Benchmarks against last release (AMD64)
+permissions: read-all
+on:
+  schedule:
+    - cron: '10 5 * * *' # runs every day at 05:10 UTC
+  # workflow_dispatch enables manual testing of this job by maintainers
+  workflow_dispatch:
+jobs:
+  amd64:
+    uses: ./.github/workflows/benchmark-template.yaml
+    with:
+      benchGitRef: release-1.3

.github/workflows/benchmark-template.yaml

@@ -0,0 +1,57 @@
+---
+name: Reusable Benchmark Template
+on:
+  workflow_call:
+    inputs:
+      # which git reference to benchmark against
+      benchGitRef:
+        required: true
+        type: string
+      maxAcceptableDifferencePercent:
+        required: false
+        type: number
+        default: 5
+      runs-on:
+        required: false
+        type: string
+        default: "['ubuntu-latest-8-cores']"
+permissions: read-all
+
+jobs:
+  benchmark:
+    runs-on: ${{ fromJson(inputs.runs-on) }}
+    steps:
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      with:
+        fetch-depth: 0
+    - id: goversion
+      run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: ${{ steps.goversion.outputs.goversion }}
+    - name: Run Benchmarks
+      run: |
+        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare REF=${{ inputs.benchGitRef }}
+    - run: |
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat result.txt >> "$GITHUB_STEP_SUMMARY"
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat <<EOL >> "$GITHUB_STEP_SUMMARY"
+        <hr />
+        The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10).
+        The last row has the Geometric Mean (geomean) for the given rows in the table.
+        Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help.
+        EOL
+    - name: Validate results under acceptable limit
+      run: |
+        export MAX_ACCEPTABLE_DIFFERENCE=${{ inputs.maxAcceptableDifferencePercent }}
+        while IFS= read -r line; do
+          # Get fourth value, which is the comparison with the base.
+          value="$(echo "$line" | awk '{print $4}')"
+          if [[ "$value" = +* ]] || [[ "$value" = -* ]]; then
+            if (( $(echo "${value//[^0-9.]/}"'>'"$MAX_ACCEPTABLE_DIFFERENCE" | bc -l) )); then
+              echo "::error::$value is above the maximum acceptable difference ($MAX_ACCEPTABLE_DIFFERENCE)"
+              exit 1
+            fi
+          fi
+        done < <(grep geomean result.txt)