ci: Continuous benchmarking with codspeed.io (#1925)

Let's see if this works with the llvm lib...

.github/workflows/ci-rs.yml

@@ -88,10 +88,12 @@ jobs:
           RUSTDOCFLAGS: "-Dwarnings"
 
   benches:
-    name: Build benchmarks 🏋️
+    name: Continuous benchmarking 🏋️
     needs: changes
     if: ${{ needs.changes.outputs.rust == 'true' && github.event_name != 'merge_group' }}
     runs-on: ubuntu-latest
+    permissions:
+      checks: write
     steps:
       - uses: actions/checkout@v4
       - uses: mozilla-actions/[email protected]
@@ -101,10 +103,18 @@ jobs:
         uses: KyleMayes/install-llvm-action@v2
         with:
           version: ${{ env.LLVM_VERSION }}
-      - name: Build benchmarks with no features
-        run: cargo bench --verbose --no-run  --no-default-features
-      - name: Build benchmarks with all features
-        run: cargo bench --verbose --no-run  --all-features
+      - uses: cargo-bins/cargo-binstall@main
+      - name: Install cargo-codspeed
+        run: cargo binstall cargo-codspeed --force
+      - name: Override criterion with the CodSpeed harness
+        run: cargo add --dev codspeed-criterion-compat --rename criterion --package hugr
+      - name: Build benchmarks
+        run: cargo codspeed build --profile bench --features extension_inference,declarative,model_unstable,llvm,llvm-test
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: "cargo codspeed run"
 
   # Run tests on Rust stable
   tests-stable-no-features:

Cargo.toml

@@ -37,7 +37,6 @@ bitvec = "1.0.1"
 capnp = "0.20.1"
 cgmath = "0.18.0"
 cool_asserts = "2.0.3"
-criterion = "0.5.1"
 delegate = "0.13.0"
 derive_more = "1.0.0"
 downcast-rs = "2.0.1"

hugr/Cargo.toml

@@ -37,10 +37,13 @@ hugr-llvm = { path = "../hugr-llvm", version = "0.14.4", optional = true }
 [dev-dependencies]
 rstest = { workspace = true }
 lazy_static = { workspace = true }
-criterion = { workspace = true, features = ["html_reports"] }
 serde_json = { workspace = true }
 bumpalo = { workspace = true, features = ["collections"] }
 
+# Defined here so it can be overridden by the codspeed CI job
+# using `cargo add`.
+criterion = { version = "0.5.1", features = ["html_reports"] }
+
 [[bench]]
 name = "bench_main"
 harness = false

hugr/benches/benchmarks/hugr/examples.rs

@@ -7,8 +7,9 @@ use hugr::builder::{
     HugrBuilder, ModuleBuilder,
 };
 use hugr::extension::prelude::{bool_t, qb_t, usize_t};
+use hugr::extension::ExtensionSet;
 use hugr::ops::OpName;
-use hugr::std_extensions::arithmetic::float_types::{float64_type, ConstF64};
+use hugr::std_extensions::arithmetic::float_types::{self, float64_type, ConstF64};
 use hugr::types::Signature;
 use hugr::{type_row, CircuitUnit, Extension, Hugr, Node};
 use lazy_static::lazy_static;
@@ -97,7 +98,10 @@ pub fn circuit(layers: usize) -> (Hugr, Vec<CircuitLayer>) {
     let cx_gate = QUANTUM_EXT.instantiate_extension_op("CX", []).unwrap();
     let rz = QUANTUM_EXT.instantiate_extension_op("Rz", []).unwrap();
     let signature =
-        Signature::new_endo(vec![qb_t(), qb_t()]).with_extension_delta(QUANTUM_EXT.name().clone());
+        Signature::new_endo(vec![qb_t(), qb_t()]).with_extension_delta(ExtensionSet::from_iter([
+            QUANTUM_EXT.name().clone(),
+            float_types::EXTENSION_ID,
+        ]));
     let mut module_builder = ModuleBuilder::new();
     let mut f_build = module_builder.define_function("main", signature).unwrap();