fix: mitigate build errors setting up cargo-binstall (#146)

fix: account for error bars in benchmark deltas

.github/workflows/ci.yaml

@@ -92,9 +92,6 @@ jobs:
         with:
           key: "${{ matrix.target }}"
 
-      - name: Install cargo-binstall
-        uses: cargo-bins/[email protected]
-
       - name: Install native prerequisites
         run: sudo apt-get update -y && sudo apt-get install -y gcc-aarch64-linux-gnu
 
@@ -131,11 +128,10 @@ jobs:
           # Needed to make sure cargo-deny is correctly cached.
           cache-all-crates: true
 
-      - name: Install cargo-binstall
-        uses: cargo-bins/[email protected]
-
       - name: Install cargo-llvm-cov
-        run: cargo binstall --no-confirm --force cargo-llvm-cov
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-llvm-cov
 
       - name: Test
         run: |
@@ -178,6 +174,11 @@ jobs:
         with:
           artifact_download_workflow_names: "CI"
           filename: "codecov.xml"
+          overall_coverage_fail_threshold: 70
+          only_list_changed_files: ${{ github.event_name == 'pull_request' }}
+          fail_on_negative_difference: true
+          negative_difference_by: "overall"
+          negative_difference_threshold: 5
 
       - name: "Upload code coverage report"
         uses: actions/upload-artifact@v4
@@ -217,20 +218,19 @@ jobs:
           # Needed to make sure cargo-deny is correctly cached.
           cache-all-crates: true
 
-      - name: Install cargo-binstall
-        uses: cargo-bins/[email protected]
-
       - name: Format check
         run: cargo fmt --check --all
 
       - name: Check
         run: cargo check --all-features --all-targets
 
+      - name: Install cargo-deny
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-deny
+
       - name: Deny check
-        run: |
-          set -euo pipefail
-          cargo binstall --no-confirm --force cargo-deny
-          cargo deny --all-features check all
+        run: cargo deny --all-features check all
 
       - name: Clippy check
         if: matrix.rust-version == 'stable'
@@ -264,9 +264,6 @@ jobs:
             ./pr
             ./main
 
-      - name: Install cargo-binstall
-        uses: cargo-bins/[email protected]
-
       - name: Performance analysis on PR
         run: cargo bench --workspace -- --output-format bencher | tee benchmarks.txt
         working-directory: pr

scripts/compare-benchmark-results.py

@@ -15,7 +15,7 @@
 class Benchmark:
     test_name: str
     duration_in_ns: int
-    deviation_in_ns: int
+    plus_or_minus_in_ns: int
 
 def parse_benchmarks_results(file_path: str) -> Dict[str, Benchmark]:
     benchmarks = {}
@@ -27,7 +27,7 @@ def parse_benchmarks_results(file_path: str) -> Dict[str, Benchmark]:
                 benchmark = Benchmark(
                     test_name=match.group(1),
                     duration_in_ns=int(match.group(2)),
-                    deviation_in_ns=int(match.group(3))
+                    plus_or_minus_in_ns=int(match.group(3))
                 )
 
                 benchmarks[benchmark.test_name] = benchmark
@@ -47,34 +47,50 @@ def parse_benchmarks_results(file_path: str) -> Dict[str, Benchmark]:
 print("# Performance Benchmark Report")
 
 if common:
-    print(f"| {'Benchmark name':36} | {'Baseline (ns)':>13} | {'Test/PR (ns)':>13} | {'Delta (ns)':>13} | {'Delta %'} |")
-    print(f"| {'-' * 36} | {'-' * 13} | {'-' * 13} | {'-' * 13} | {'-' * 7}")
+    print(f"| {'Benchmark name':38} | {'Baseline (μs)':>13} | {'Test/PR (μs)':>13} | {'Delta (μs)':>13} | {'Delta %':15} |")
+    print(f"| {'-' * 38} | {'-' * 13} | {'-' * 13} | {'-' * 13} | {'-' * 15} |")
     for name in sorted(common):
-        base_duration = base_results[name].duration_in_ns
-        test_duration = test_results[name].duration_in_ns
+        # Retrieve base data
+        base_duration = base_results[name].duration_in_ns / 1000.0
+        base_plus_or_minus = base_results[name].plus_or_minus_in_ns / 1000.0
+        base_plus_or_minus_percentage = (100.0 * base_plus_or_minus) / base_duration
 
-        delta_duration = test_duration - base_duration
-        delta_str = str(delta_duration)
-        if delta_duration > 0:
-            delta_str = "+" + delta_str
+        # Retrieve test data
+        test_duration = test_results[name].duration_in_ns / 1000.0
+        test_plus_or_minus = test_results[name].plus_or_minus_in_ns / 1000.0
+        test_plus_or_minus_percentage = (100.0 * test_plus_or_minus) / test_duration
 
+        # Compute delta
+        delta_duration = test_duration - base_duration
         delta_percentage = (100.0 * delta_duration) / base_duration
-        delta_percentage_str = f"{delta_percentage:.2f}%"
-        if delta_percentage < 0:
-            delta_percentage_str = "🟢 " + delta_percentage_str
-        elif delta_percentage > 0:
-            delta_percentage_str = "🟠 +" + delta_percentage_str
+        abs_delta_percentage = abs(delta_percentage)
+        max_plus_or_minus_percentage = max(base_plus_or_minus_percentage, test_plus_or_minus_percentage)
+
+        # Format
+        delta_str = f"{delta_duration:8.2f}"
+
+        if abs_delta_percentage > max_plus_or_minus_percentage:
+            if delta_percentage < 0:
+                delta_prefix = "🟢 "
+            elif delta_percentage > 0:
+                delta_prefix = "🟠 +"
+            else:
+                delta_prefix = "⚪  "
+
+            delta_percentage_str = f"{delta_prefix}{delta_percentage:.2f}%"
         else:
-            delta_percentage_str = "⚪  " + delta_percentage_str
+            delta_percentage_str = "⚪  Unchanged"
 
-        print(f"| `{name:36}` | `{base_duration:10} ns` | `{test_duration:10} ns` | `{delta_str:>10} ns` | `{delta_percentage_str:>7}` |")
+        print(f"| `{name:36}` | `{base_duration:8.2f} μs` | `{test_duration:8.2f} μs` | `{delta_str:>8} μs` | `{delta_percentage_str:12}` |")
 
 if removed_from_base:
+    print()
     print("Benchmarks removed:")
     for name in removed_from_base:
         print(f"  - {name}")
 
 if added_by_test:
+    print()
     print("Benchmarks added:")
     for name in added_by_test:
         print(f"  - {name}")