diff --git a/.circleci/config.yml b/.circleci/config.yml index 9262050f73..323035270d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,8 +84,7 @@ workflows: - main - /^[0-9]+\.[0-9]+$/ # Add your branch here if benchmarking matters to your work - - fix-benchmarking - - w3 + - add-noop - coverage deploy: jobs: diff --git a/contracts/README.md b/contracts/README.md index 84795914e2..c9f6dd2131 100644 --- a/contracts/README.md +++ b/contracts/README.md @@ -64,6 +64,11 @@ docker run --rm -v "$(pwd)":/code \ --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \ cosmwasm/optimizer:0.15.0 ./contracts/crypto-verify +docker run --rm -v "$(pwd)":/code \ + --mount type=volume,source="devcontract_cache_cyberpunk",target=/target \ + --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \ + cosmwasm/optimizer:0.15.0 ./contracts/cyberpunk + docker run --rm -v "$(pwd)":/code \ --mount type=volume,source="devcontract_cache_floaty",target=/target \ --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \ diff --git a/contracts/cyberpunk/schema/cyberpunk.json b/contracts/cyberpunk/schema/cyberpunk.json index ccb550b1ad..0660511517 100644 --- a/contracts/cyberpunk/schema/cyberpunk.json +++ b/contracts/cyberpunk/schema/cyberpunk.json @@ -179,6 +179,20 @@ } }, "additionalProperties": false + }, + { + "description": "Does nothing. This can be used for baseline contract execution performance measurements.", + "type": "object", + "required": [ + "noop" + ], + "properties": { + "noop": { + "type": "object", + "additionalProperties": false + } + }, + "additionalProperties": false } ] }, diff --git a/contracts/cyberpunk/schema/raw/execute.json b/contracts/cyberpunk/schema/raw/execute.json index 8ea521b97f..eeb2687ead 100644 --- a/contracts/cyberpunk/schema/raw/execute.json +++ b/contracts/cyberpunk/schema/raw/execute.json @@ -169,6 +169,20 @@ } }, "additionalProperties": false + }, + { + "description": "Does nothing. This can be used for baseline contract execution performance measurements.", + "type": "object", + "required": [ + "noop" + ], + "properties": { + "noop": { + "type": "object", + "additionalProperties": false + } + }, + "additionalProperties": false } ] } diff --git a/contracts/cyberpunk/src/contract.rs b/contracts/cyberpunk/src/contract.rs index 4972297e00..80254fe64d 100644 --- a/contracts/cyberpunk/src/contract.rs +++ b/contracts/cyberpunk/src/contract.rs @@ -39,6 +39,7 @@ pub fn execute( Unreachable {} => execute_unreachable(), MirrorEnv {} => execute_mirror_env(env), Debug {} => execute_debug(deps.api), + Noop {} => execute_noop(), } } @@ -178,6 +179,10 @@ fn execute_debug(api: &dyn Api) -> Result { Ok(Response::default()) } +fn execute_noop() -> Result { + Ok(Response::new()) +} + #[entry_point] pub fn query(deps: Deps, env: Env, msg: QueryMsg) -> StdResult { use QueryMsg::*; diff --git a/contracts/cyberpunk/src/msg.rs b/contracts/cyberpunk/src/msg.rs index c02b518721..6dc9febca6 100644 --- a/contracts/cyberpunk/src/msg.rs +++ b/contracts/cyberpunk/src/msg.rs @@ -30,6 +30,8 @@ pub enum ExecuteMsg { MirrorEnv {}, /// Does a bit of work and calls debug Debug {}, + /// Does nothing. This can be used for baseline contract execution performance measurements. + Noop {}, } #[cw_serde] diff --git a/packages/vm/benches/main.rs b/packages/vm/benches/main.rs index 9f47467131..f925418b75 100644 --- a/packages/vm/benches/main.rs +++ b/packages/vm/benches/main.rs @@ -30,7 +30,8 @@ const MEMORY_CACHE_SIZE: Size = Size::mebi(200); const INSTANTIATION_THREADS: usize = 128; const CONTRACTS: u64 = 10; -static CONTRACT: &[u8] = include_bytes!("../testdata/hackatom.wasm"); +const DEFAULT_CAPABILITIES: &str = "cosmwasm_1_1,cosmwasm_1_2,cosmwasm_1_3,iterator,staking"; +static HACKATOM: &[u8] = include_bytes!("../testdata/hackatom.wasm"); static CYBERPUNK: &[u8] = include_bytes!("../testdata/cyberpunk.wasm"); fn bench_instance(c: &mut Criterion) { @@ -41,7 +42,7 @@ fn bench_instance(c: &mut Criterion) { let backend = mock_backend(&[]); let (instance_options, memory_limit) = mock_instance_options(); let _instance = - Instance::from_code(CONTRACT, backend, instance_options, memory_limit).unwrap(); + Instance::from_code(HACKATOM, backend, instance_options, memory_limit).unwrap(); }); }); @@ -51,7 +52,7 @@ fn bench_instance(c: &mut Criterion) { gas_limit: HIGH_GAS_LIMIT, }; let mut instance = - Instance::from_code(CONTRACT, backend, much_gas, Some(DEFAULT_MEMORY_LIMIT)).unwrap(); + Instance::from_code(HACKATOM, backend, much_gas, Some(DEFAULT_MEMORY_LIMIT)).unwrap(); b.iter(|| { let info = mock_info(&instance.api().addr_make("creator"), &coins(1000, "earth")); @@ -75,7 +76,7 @@ fn bench_instance(c: &mut Criterion) { gas_limit: HIGH_GAS_LIMIT, }; let mut instance = - Instance::from_code(CONTRACT, backend, much_gas, Some(DEFAULT_MEMORY_LIMIT)).unwrap(); + Instance::from_code(HACKATOM, backend, much_gas, Some(DEFAULT_MEMORY_LIMIT)).unwrap(); let info = mock_info(&instance.api().addr_make("creator"), &coins(1000, "earth")); let verifier = instance.api().addr_make("verifies"); @@ -129,7 +130,7 @@ fn bench_cache(c: &mut Criterion) { let options = CacheOptions::new( TempDir::new().unwrap().into_path(), - capabilities_from_csv("iterator,staking"), + capabilities_from_csv(DEFAULT_CAPABILITIES), MEMORY_CACHE_SIZE, DEFAULT_MEMORY_LIMIT, ); @@ -139,7 +140,7 @@ fn bench_cache(c: &mut Criterion) { unsafe { Cache::new(options.clone()).unwrap() }; b.iter(|| { - let result = cache.save_wasm(CONTRACT); + let result = cache.save_wasm(HACKATOM); assert!(result.is_ok()); }); }); @@ -147,7 +148,7 @@ fn bench_cache(c: &mut Criterion) { group.bench_function("load wasm", |b| { let cache: Cache = unsafe { Cache::new(options.clone()).unwrap() }; - let checksum = cache.save_wasm(CONTRACT).unwrap(); + let checksum = cache.save_wasm(HACKATOM).unwrap(); b.iter(|| { let result = cache.load_wasm(&checksum); @@ -160,7 +161,7 @@ fn bench_cache(c: &mut Criterion) { let mut cache: Cache = unsafe { Cache::new(options).unwrap() }; cache.set_module_unchecked(true); - let checksum = cache.save_wasm(CONTRACT).unwrap(); + let checksum = cache.save_wasm(HACKATOM).unwrap(); b.iter(|| { let result = cache.load_wasm(&checksum); @@ -171,7 +172,7 @@ fn bench_cache(c: &mut Criterion) { group.bench_function("analyze", |b| { let cache: Cache = unsafe { Cache::new(options.clone()).unwrap() }; - let checksum = cache.save_wasm(CONTRACT).unwrap(); + let checksum = cache.save_wasm(HACKATOM).unwrap(); b.iter(|| { let result = cache.analyze(&checksum); @@ -182,13 +183,13 @@ fn bench_cache(c: &mut Criterion) { group.bench_function("instantiate from fs", |b| { let non_memcache = CacheOptions::new( TempDir::new().unwrap().into_path(), - capabilities_from_csv("iterator,staking"), + capabilities_from_csv(DEFAULT_CAPABILITIES), Size::new(0), DEFAULT_MEMORY_LIMIT, ); let cache: Cache = unsafe { Cache::new(non_memcache).unwrap() }; - let checksum = cache.save_wasm(CONTRACT).unwrap(); + let checksum = cache.save_wasm(HACKATOM).unwrap(); b.iter(|| { let _ = cache @@ -204,14 +205,14 @@ fn bench_cache(c: &mut Criterion) { group.bench_function("instantiate from fs unchecked", |b| { let non_memcache = CacheOptions::new( TempDir::new().unwrap().into_path(), - capabilities_from_csv("iterator,staking"), + capabilities_from_csv(DEFAULT_CAPABILITIES), Size::new(0), DEFAULT_MEMORY_LIMIT, ); let mut cache: Cache = unsafe { Cache::new(non_memcache).unwrap() }; cache.set_module_unchecked(true); - let checksum = cache.save_wasm(CONTRACT).unwrap(); + let checksum = cache.save_wasm(HACKATOM).unwrap(); b.iter(|| { let _ = cache @@ -225,7 +226,7 @@ fn bench_cache(c: &mut Criterion) { }); group.bench_function("instantiate from memory", |b| { - let checksum = Checksum::generate(CONTRACT); + let checksum = Checksum::generate(HACKATOM); let cache: Cache = unsafe { Cache::new(options.clone()).unwrap() }; // Load into memory @@ -246,7 +247,7 @@ fn bench_cache(c: &mut Criterion) { }); group.bench_function("instantiate from pinned memory", |b| { - let checksum = Checksum::generate(CONTRACT); + let checksum = Checksum::generate(HACKATOM); let cache: Cache = unsafe { Cache::new(options.clone()).unwrap() }; // Load into pinned memory @@ -267,11 +268,11 @@ fn bench_cache(c: &mut Criterion) { group.finish(); } -pub fn bench_instance_threads(c: &mut Criterion) { +fn bench_instance_threads(c: &mut Criterion) { c.bench_function("multi-threaded get_instance", |b| { let options = CacheOptions::new( TempDir::new().unwrap().into_path(), - capabilities_from_csv("iterator,staking"), + capabilities_from_csv(DEFAULT_CAPABILITIES), MEMORY_CACHE_SIZE, DEFAULT_MEMORY_LIMIT, ); @@ -290,10 +291,10 @@ pub fn bench_instance_threads(c: &mut Criterion) { // Offset to the i32.const (0x41) 15731626 (0xf00baa) (unsigned leb128 encoded) instruction // data we want to replace let query_int_data = b"\x41\xaa\x97\xc0\x07"; - let offset = find_subsequence(CONTRACT, query_int_data).unwrap() + 1; + let offset = find_subsequence(HACKATOM, query_int_data).unwrap() + 1; let mut leb128_buf = [0; 4]; - let mut contract = CONTRACT.to_vec(); + let mut contract = HACKATOM.to_vec(); let mut random_checksum = || { let mut writable = &mut leb128_buf[..]; @@ -352,24 +353,127 @@ pub fn bench_instance_threads(c: &mut Criterion) { }); } -fn make_config() -> Criterion { +fn bench_combined(c: &mut Criterion) { + let mut group = c.benchmark_group("Combined"); + + let options = CacheOptions::new( + TempDir::new().unwrap().into_path(), + capabilities_from_csv("cosmwasm_1_1,cosmwasm_1_2,cosmwasm_1_3,iterator,staking"), + MEMORY_CACHE_SIZE, + DEFAULT_MEMORY_LIMIT, + ); + + // Store contracts for all benchmarks in this group + let checksum: Checksum = { + let cache: Cache = + unsafe { Cache::new(options.clone()).unwrap() }; + cache.save_wasm(CYBERPUNK).unwrap() + }; + + group.bench_function("get instance from fs cache and execute", |b| { + let mut non_memcache = options.clone(); + non_memcache.memory_cache_size = Size::kibi(0); + + let cache: Cache = + unsafe { Cache::new(non_memcache).unwrap() }; + + b.iter(|| { + let mut instance = cache + .get_instance(&checksum, mock_backend(&[]), DEFAULT_INSTANCE_OPTIONS) + .unwrap(); + assert_eq!(cache.stats().hits_pinned_memory_cache, 0); + assert_eq!(cache.stats().hits_memory_cache, 0); + assert!(cache.stats().hits_fs_cache >= 1); + assert_eq!(cache.stats().misses, 0); + + let info = mock_info("guest", &[]); + let msg = br#"{"noop":{}}"#; + let contract_result = + call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg).unwrap(); + contract_result.into_result().unwrap(); + }); + }); + + group.bench_function("get instance from memory cache and execute", |b| { + let cache: Cache = + unsafe { Cache::new(options.clone()).unwrap() }; + + // Load into memory + cache + .get_instance(&checksum, mock_backend(&[]), DEFAULT_INSTANCE_OPTIONS) + .unwrap(); + + b.iter(|| { + let backend = mock_backend(&[]); + let mut instance = cache + .get_instance(&checksum, backend, DEFAULT_INSTANCE_OPTIONS) + .unwrap(); + assert_eq!(cache.stats().hits_pinned_memory_cache, 0); + assert!(cache.stats().hits_memory_cache >= 1); + assert_eq!(cache.stats().hits_fs_cache, 1); + assert_eq!(cache.stats().misses, 0); + + let info = mock_info("guest", &[]); + let msg = br#"{"noop":{}}"#; + let contract_result = + call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg).unwrap(); + contract_result.into_result().unwrap(); + }); + }); + + group.bench_function("get instance from pinned memory and execute", |b| { + let cache: Cache = + unsafe { Cache::new(options.clone()).unwrap() }; + + // Load into pinned memory + cache.pin(&checksum).unwrap(); + + b.iter(|| { + let backend = mock_backend(&[]); + let mut instance = cache + .get_instance(&checksum, backend, DEFAULT_INSTANCE_OPTIONS) + .unwrap(); + assert_eq!(cache.stats().hits_memory_cache, 0); + assert!(cache.stats().hits_pinned_memory_cache >= 1); + assert_eq!(cache.stats().hits_fs_cache, 1); + assert_eq!(cache.stats().misses, 0); + + let info = mock_info("guest", &[]); + let msg = br#"{"noop":{}}"#; + let contract_result = + call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg).unwrap(); + contract_result.into_result().unwrap(); + }); + }); + + group.finish(); +} + +fn make_config(measurement_time_s: u64) -> Criterion { Criterion::default() .without_plots() - .measurement_time(Duration::new(10, 0)) + .measurement_time(Duration::new(measurement_time_s, 0)) .sample_size(12) .configure_from_args() } criterion_group!( name = instance; - config = make_config(); + config = make_config(8); targets = bench_instance ); criterion_group!( name = cache; - config = make_config(); + config = make_config(8); targets = bench_cache ); +// Combines loading module from cache, instantiating it and executing the instance. +// This is what every call in libwasmvm does. +criterion_group!( + name = combined; + config = make_config(5); + targets = bench_combined +); criterion_group!( name = multi_threaded_instance; config = Criterion::default() @@ -379,4 +483,4 @@ criterion_group!( .configure_from_args(); targets = bench_instance_threads ); -criterion_main!(instance, cache, multi_threaded_instance); +criterion_main!(instance, cache, combined, multi_threaded_instance); diff --git a/packages/vm/src/calls.rs b/packages/vm/src/calls.rs index 1b9fd3f1a6..f09e9841c4 100644 --- a/packages/vm/src/calls.rs +++ b/packages/vm/src/calls.rs @@ -692,9 +692,10 @@ mod tests { call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg).unwrap_err(); match err { VmError::RuntimeErr { msg, .. } => { - assert!(msg.contains( - "RuntimeError: Aborted: panicked at 'This page intentionally faulted'" - )) + assert!( + msg.contains("RuntimeError: Aborted: panicked at src/contract.rs:"), + "Unexpected error msg: {msg}" + ) } err => panic!("Unexpected error: {err:?}"), } diff --git a/packages/vm/src/compatibility.rs b/packages/vm/src/compatibility.rs index 4850cf3006..8bbda004c9 100644 --- a/packages/vm/src/compatibility.rs +++ b/packages/vm/src/compatibility.rs @@ -277,7 +277,7 @@ mod tests { static CONTRACT_RUST_170: &[u8] = include_bytes!("../testdata/cyberpunk_rust170.wasm"); fn default_capabilities() -> HashSet { - capabilities_from_csv("cosmwasm_1_1,cosmwasm_1_2,iterator,staking,stargate") + capabilities_from_csv("cosmwasm_1_1,cosmwasm_1_2,cosmwasm_1_3,iterator,staking,stargate") } #[test] diff --git a/packages/vm/testdata/cyberpunk.wasm b/packages/vm/testdata/cyberpunk.wasm index 68aa44257f..885a1dbdf3 100644 Binary files a/packages/vm/testdata/cyberpunk.wasm and b/packages/vm/testdata/cyberpunk.wasm differ